Search in sources :

Example 1 with MantisWorkerMetadataWritable

use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.

the class JobRouteTest method testJobClusterGetJobsList.

@Test(dependsOnMethods = { "testJobClusterGetJobIds" })
public void testJobClusterGetJobsList() throws InterruptedException {
    final CountDownLatch latch = new CountDownLatch(1);
    final CompletionStage<HttpResponse> responseFuture = http.singleRequest(HttpRequest.GET(jobAPIEndpoint("list")));
    responseFuture.thenCompose(r -> processRespFut(r, Optional.of(200))).whenComplete((msg, t) -> {
        String responseMessage = getResponseMessage(msg, t);
        logger.info("got response---> {}", responseMessage);
        List<MantisJobMetadataView> jobInfos = Collections.emptyList();
        try {
            jobInfos = Jackson.fromJSON(responseMessage, new TypeReference<List<MantisJobMetadataView>>() {
            });
        } catch (IOException e) {
            logger.error("failed to deser json {}", responseMessage, e);
            fail("job list deser failed");
        }
        logger.info("jobInfos---> {}", jobInfos);
        assertEquals(1, jobInfos.size());
        MantisJobMetadataView mjm = jobInfos.get(0);
        assertEquals(mjm.getJobMetadata().getJobId(), "sine-function-1");
        assertEquals(mjm.getJobMetadata().getName(), "sine-function");
        assertTrue(mjm.getStageMetadataList().size() > 0);
        MantisStageMetadataWritable msm = mjm.getStageMetadataList().get(0);
        assertEquals(1, msm.getNumWorkers());
        assertTrue(mjm.getWorkerMetadataList().size() > 0);
        MantisWorkerMetadataWritable mwm = mjm.getWorkerMetadataList().get(0);
        assertEquals("sine-function-1", mwm.getJobId());
        assertEquals(false, mwm.getCluster().isPresent());
        latch.countDown();
    });
    assertTrue(latch.await(2, TimeUnit.SECONDS));
}
Also used : JobStatusStreamRoute(io.mantisrx.master.api.akka.route.v1.JobStatusStreamRoute) TestHelpers(com.netflix.mantis.master.scheduler.TestHelpers) MantisJobDurationType(io.mantisrx.runtime.MantisJobDurationType) MantisJobState(io.mantisrx.runtime.MantisJobState) MasterDescription(io.mantisrx.server.core.master.MasterDescription) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) TypeReference(io.mantisrx.shaded.com.fasterxml.jackson.core.type.TypeReference) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) Test(org.testng.annotations.Test) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) JobsRoute(io.mantisrx.master.api.akka.route.v1.JobsRoute) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) JobClustersManagerActor(io.mantisrx.master.JobClustersManagerActor) JobPayloads(io.mantisrx.master.api.akka.payloads.JobPayloads) ActorMaterializer(akka.stream.ActorMaterializer) ActorRef(akka.actor.ActorRef) MantisMasterRoute(io.mantisrx.master.api.akka.route.MantisMasterRoute) Duration(java.time.Duration) Map(java.util.Map) JobClusterProtoAdapter(io.mantisrx.master.api.akka.route.proto.JobClusterProtoAdapter) Assert.fail(org.junit.Assert.fail) LastSubmittedJobIdStreamRoute(io.mantisrx.master.api.akka.route.v1.LastSubmittedJobIdStreamRoute) StatusEventSubscriberLoggingImpl(io.mantisrx.master.events.StatusEventSubscriberLoggingImpl) ServerBinding(akka.http.javadsl.ServerBinding) HttpCharsets(akka.http.javadsl.model.HttpCharsets) JobStatusRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobStatusRouteHandler) WorkerEventSubscriberLoggingImpl(io.mantisrx.master.events.WorkerEventSubscriberLoggingImpl) Jackson(io.mantisrx.master.api.akka.route.Jackson) BeforeClass(org.testng.annotations.BeforeClass) AdminMasterRoute(io.mantisrx.master.api.akka.route.v1.AdminMasterRoute) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) HttpMethods(akka.http.javadsl.model.HttpMethods) WorkerAssignments(io.mantisrx.server.core.WorkerAssignments) JobClusterRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) AgentClusterOperations(io.mantisrx.master.vm.AgentClusterOperations) CompletionStage(java.util.concurrent.CompletionStage) NotUsed(akka.NotUsed) AgentClustersRoute(io.mantisrx.master.api.akka.route.v1.AgentClustersRoute) ActorSystem(akka.actor.ActorSystem) JobDiscoveryStreamRoute(io.mantisrx.master.api.akka.route.v1.JobDiscoveryStreamRoute) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) Optional(java.util.Optional) Mockito.mock(org.mockito.Mockito.mock) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) Flow(akka.stream.javadsl.Flow) JobRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobRouteHandlerAkkaImpl) LeaderRedirectionFilter(io.mantisrx.server.master.LeaderRedirectionFilter) Matchers.anyString(org.mockito.Matchers.anyString) AuditEventSubscriberLoggingImpl(io.mantisrx.master.events.AuditEventSubscriberLoggingImpl) Observable(rx.Observable) JobClusterPayloads(io.mantisrx.master.api.akka.payloads.JobClusterPayloads) CompactJobInfo(io.mantisrx.server.master.http.api.CompactJobInfo) LifecycleEventPublisherImpl(io.mantisrx.master.events.LifecycleEventPublisherImpl) JobTestHelper(io.mantisrx.master.jobcluster.job.JobTestHelper) ByteString(akka.util.ByteString) HttpEntity(akka.http.javadsl.model.HttpEntity) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) LocalMasterMonitor(io.mantisrx.server.core.master.LocalMasterMonitor) LeadershipManagerLocalImpl(io.mantisrx.server.master.LeadershipManagerLocalImpl) JobClusterRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandlerAkkaImpl) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) AfterClass(org.testng.annotations.AfterClass) Logger(org.slf4j.Logger) Assert.assertNotNull(org.junit.Assert.assertNotNull) Http(akka.http.javadsl.Http) HttpRequest(akka.http.javadsl.model.HttpRequest) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) ContentTypes(akka.http.javadsl.model.ContentTypes) HttpEntities(akka.http.javadsl.model.HttpEntities) HttpResponse(akka.http.javadsl.model.HttpResponse) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) ConnectHttp(akka.http.javadsl.ConnectHttp) MediaTypes(akka.http.javadsl.model.MediaTypes) JobClustersRoute(io.mantisrx.master.api.akka.route.v1.JobClustersRoute) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) HttpResponse(akka.http.javadsl.model.HttpResponse) Matchers.anyString(org.mockito.Matchers.anyString) ByteString(akka.util.ByteString) TypeReference(io.mantisrx.shaded.com.fasterxml.jackson.core.type.TypeReference) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Example 2 with MantisWorkerMetadataWritable

use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.

the class JacksonTest method testDeser4.

@Test
public void testDeser4() throws IOException {
    final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    List<MantisJobMetadataView> jobIdInfos = Jackson.fromJSON(objectMapper, "[{\"jobMetadata\":{\"jobId\":\"sine-function-1\",\"name\":\"sine-function\"," + "\"user\":\"nmahilani\",\"submittedAt\":1527703650220,\"jarUrl\":\"https://mantis.staging.us-east-1.prod.netflix.net/mantis-artifacts/mantis-examples-sine-function-0.2.9.zip\"," + "\"numStages\":2,\"sla\":{\"runtimeLimitSecs\":0,\"minRuntimeSecs\":0,\"slaType\":\"Lossy\",\"durationType\":\"Perpetual\",\"userProvidedType\":\"\"}," + "\"state\":\"Accepted\",\"subscriptionTimeoutSecs\":0,\"parameters\":[{\"name\":\"useRandom\",\"value\":\"True\"}],\"nextWorkerNumberToUse\":11," + "\"migrationConfig\":{\"strategy\":\"PERCENTAGE\",\"configString\":\"{\\\"percentToMove\\\":25,\\\"intervalMs\\\":60000}\"}," + "\"labels\":[{\"name\":\"_mantis.user\",\"value\":\"nmahilani\"},{\"name\":\"_mantis.ownerEmail\",\"value\":\"nmahilani@netflix.com\"}," + "{\"name\":\"_mantis.jobType\",\"value\":\"other\"},{\"name\":\"_mantis.criticality\",\"value\":\"low\"},{\"name\":\"_mantis.artifact.version\",\"value\":\"0.2.9\"}]}," + "\"stageMetadataList\":[{\"jobId\":\"sine-function-1\",\"stageNum\":0,\"numStages\":2,\"machineDefinition\":{\"cpuCores\":1.0,\"memoryMB\":200.0,\"networkMbps\":128.0,\"diskMB\":1024.0,\"numPorts\":1}," + "\"numWorkers\":1,\"hardConstraints\":null,\"softConstraints\":null,\"scalingPolicy\":null,\"scalable\":false}," + "{\"jobId\":\"sine-function-1\",\"stageNum\":1,\"numStages\":2,\"machineDefinition\":{\"cpuCores\":1.0,\"memoryMB\":200.0,\"networkMbps\":128.0,\"diskMB\":1024.0,\"numPorts\":1},\"numWorkers\":1,\"hardConstraints\":[],\"softConstraints\":[\"M4Cluster\"]," + "\"scalingPolicy\":{\"stage\":1,\"min\":1,\"max\":10,\"increment\":2,\"decrement\":1,\"coolDownSecs\":600," + "\"strategies\":{\"CPU\":{\"reason\":\"CPU\",\"scaleDownBelowPct\":15.0,\"scaleUpAbovePct\":75.0,\"rollingCount\":{\"count\":12,\"of\":20}}},\"enabled\":true},\"scalable\":true}]," + "\"workerMetadataList\":[{\"workerIndex\":0,\"workerNumber\":2,\"jobId\":\"sine-function-1\",\"stageNum\":0,\"numberOfPorts\":4,\"metricsPort\":0,\"consolePort\":0," + "\"debugPort\":-1,\"ports\":[],\"state\":\"Accepted\",\"slave\":null,\"slaveID\":null,\"cluster\":{\"present\":false},\"acceptedAt\":1527703650231,\"launchedAt\":0,\"startingAt\":0,\"startedAt\":0," + "\"completedAt\":0,\"reason\":null,\"resubmitOf\":-1,\"totalResubmitCount\":0},{\"workerIndex\":0,\"workerNumber\":3,\"jobId\":\"sine-function-1\",\"stageNum\":1,\"numberOfPorts\":4,\"metricsPort\":0,\"consolePort\":0,\"debugPort\":-1,\"ports\":[],\"state\":\"Accepted\"," + "\"slave\":null,\"slaveID\":null,\"cluster\":{\"present\":false},\"acceptedAt\":1527703650232,\"launchedAt\":0,\"startingAt\":0,\"startedAt\":0,\"completedAt\":0," + "\"reason\":null,\"resubmitOf\":-1,\"totalResubmitCount\":0}]}]", new TypeReference<List<MantisJobMetadataView>>() {
    });
    System.out.println(jobIdInfos);
    MantisWorkerMetadataWritable mwm = jobIdInfos.get(0).getWorkerMetadataList().get(0);
    mwm.setCluster(Optional.ofNullable("test"));
    System.out.println(objectMapper.writer(Jackson.DEFAULT_FILTER_PROVIDER).writeValueAsString(mwm));
}
Also used : MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) List(java.util.List) ObjectMapper(io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper) Test(org.junit.Test)

Example 3 with MantisWorkerMetadataWritable

use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.

the class NoOpMantisJobOperations method convertMantisWorkerMetadataToMantisWorkerMetadataWritable.

public static MantisWorkerMetadataWritable convertMantisWorkerMetadataToMantisWorkerMetadataWritable(IMantisWorkerMetadata workerMeta) {
    MantisWorkerMetadataWritable writable = new MantisWorkerMetadataWritable(workerMeta.getWorkerIndex(), workerMeta.getWorkerNumber(), workerMeta.getJobId(), workerMeta.getStageNum(), workerMeta.getNumberOfPorts());
    setWorkerMetadataWritable(writable, workerMeta);
    return writable;
}
Also used : FilterableMantisWorkerMetadataWritable(io.mantisrx.master.jobcluster.job.FilterableMantisWorkerMetadataWritable) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable)

Example 4 with MantisWorkerMetadataWritable

use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.

the class JobRoute method getJobRoutes.

private Route getJobRoutes() {
    return route(path(STATUS_ENDPOINT, () -> post(() -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
        if (logger.isDebugEnabled()) {
            logger.debug("/api/postjobstatus called {}", req);
        }
        try {
            workerHeartbeatStatusPOST.increment();
            PostJobStatusRequest postJobStatusRequest = Jackson.fromJSON(req, PostJobStatusRequest.class);
            WorkerEvent workerStatusRequest = createWorkerStatusRequest(postJobStatusRequest);
            if (workerStatusRequest instanceof WorkerHeartbeat) {
                if (!ConfigurationProvider.getConfig().isHeartbeatProcessingEnabled()) {
                    // skip heartbeat processing
                    if (logger.isTraceEnabled()) {
                        logger.trace("skipped heartbeat event {}", workerStatusRequest);
                    }
                    workerHeartbeatSkipped.increment();
                    return complete(StatusCodes.OK);
                }
            }
            return completeWithFuture(jobRouteHandler.workerStatus(workerStatusRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error handling job status {}", req, e);
            return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid JSON payload to post job status\"}");
        }
    })))), pathPrefix(API_JOBS, () -> route(post(() -> route(path(KILL_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
        logger.debug("/api/jobs/kill called {}", req);
        try {
            final KillJobRequest killJobRequest = Jackson.fromJSON(req, KillJobRequest.class);
            return completeWithFuture(jobRouteHandler.kill(killJobRequest).thenApply(resp -> {
                if (resp.responseCode == BaseResponse.ResponseCode.SUCCESS) {
                    return new JobClusterManagerProto.KillJobResponse(resp.requestId, resp.responseCode, resp.getState(), "[\"" + resp.getJobId().getId() + " Killed\"]", resp.getJobId(), resp.getUser());
                } else if (resp.responseCode == BaseResponse.ResponseCode.CLIENT_ERROR) {
                    // for backwards compatibility with old master
                    return new JobClusterManagerProto.KillJobResponse(resp.requestId, BaseResponse.ResponseCode.SUCCESS, resp.getState(), "[\"" + resp.message + " \"]", resp.getJobId(), resp.getUser());
                }
                return resp;
            }).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error on job kill {}", req, e);
            return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to kill job\"}");
        }
    }))), path(RESUBMIT_WORKER_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
        logger.debug("/api/jobs/resubmitWorker called {}", req);
        try {
            final ResubmitWorkerRequest resubmitWorkerRequest = Jackson.fromJSON(req, ResubmitWorkerRequest.class);
            return completeWithFuture(jobRouteHandler.resubmitWorker(resubmitWorkerRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error on worker resubmit {}", req, e);
            return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to resubmit worker\"}");
        }
    }))), path(SCALE_STAGE_ENDPOINT, () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), req -> {
        logger.debug("/api/jobs/scaleStage called {}", req);
        try {
            ScaleStageRequest scaleStageRequest = Jackson.fromJSON(req, ScaleStageRequest.class);
            int numWorkers = scaleStageRequest.getNumWorkers();
            int maxWorkersPerStage = ConfigurationProvider.getConfig().getMaxWorkersPerStage();
            if (numWorkers > maxWorkersPerStage) {
                logger.warn("rejecting ScaleStageRequest {} with invalid num workers", scaleStageRequest);
                return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"num workers must be less than " + maxWorkersPerStage + "\"}");
            }
            return completeWithFuture(jobRouteHandler.scaleStage(scaleStageRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error scaling stage {}", req, e);
            return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"invalid json payload to scale stage " + e.getMessage() + "\"}");
        }
    }))))), get(() -> route(// - optional labels.op query param - default value is 'or' if not specified (other possible value is 'and'
    path(segment("list"), () -> {
        jobListGET.increment();
        return jobListRoute(Optional.empty());
    }), path(segment("list").slash("matchinglabels"), () -> {
        jobListLabelMatchGET.increment();
        return jobListRoute(Optional.empty());
    }), path(segment("list").slash(PathMatchers.segment()), (jobId) -> {
        logger.debug("/api/jobs/list/{} called", jobId);
        jobListJobIdGET.increment();
        return completeAsync(jobRouteHandler.getJobDetails(new JobClusterManagerProto.GetJobDetailsRequest("masterAPI", jobId)), resp -> {
            Optional<MantisJobMetadataView> mantisJobMetadataView = resp.getJobMetadata().map(metaData -> new MantisJobMetadataView(metaData, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), false));
            return completeOK(mantisJobMetadataView, Jackson.marshaller());
        });
    }), path(segment("list").slash("matching").slash(PathMatchers.segment()), (regex) -> {
        jobListRegexGET.increment();
        return jobListRoute(Optional.ofNullable(regex).filter(r -> !r.isEmpty()));
    }), path(segment("archived").slash(PathMatchers.segment()), (jobId) -> parameterOptional(StringUnmarshallers.INTEGER, "limit", (limit) -> {
        jobArchivedWorkersGET.increment();
        Optional<JobId> jobIdO = JobId.fromId(jobId);
        if (jobIdO.isPresent()) {
            ListArchivedWorkersRequest req = new ListArchivedWorkersRequest(jobIdO.get(), limit.orElse(DEFAULT_LIST_ARCHIVED_WORKERS_LIMIT));
            return alwaysCache(cache, requestUriKeyer, () -> extractUri(uri -> completeAsync(jobRouteHandler.listArchivedWorkers(req), resp -> {
                List<MantisWorkerMetadataWritable> workers = resp.getWorkerMetadata().stream().map(wm -> DataFormatAdapter.convertMantisWorkerMetadataToMantisWorkerMetadataWritable(wm)).collect(Collectors.toList());
                return completeOK(workers, Jackson.marshaller());
            })));
        } else {
            return complete(StatusCodes.BAD_REQUEST, "error: 'archived/<jobId>' request must include a valid jobId");
        }
    })), path(segment("archived"), () -> {
        jobArchivedWorkersGETInvalid.increment();
        return complete(StatusCodes.BAD_REQUEST, "error: 'archived' Request must include jobId");
    }))))));
}
Also used : JobId(io.mantisrx.server.master.domain.JobId) Uri(akka.http.javadsl.model.Uri) MasterConfiguration(io.mantisrx.server.master.config.MasterConfiguration) Arrays(java.util.Arrays) JavaPartialFunction(akka.japi.JavaPartialFunction) Cache(akka.http.caching.javadsl.Cache) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) LoggerFactory(org.slf4j.LoggerFactory) RequestContext(akka.http.javadsl.server.RequestContext) PathMatchers.segment(akka.http.javadsl.server.PathMatchers.segment) StringUnmarshallers(akka.http.javadsl.unmarshalling.StringUnmarshallers) PathMatcher0(akka.http.javadsl.server.PathMatcher0) ListArchivedWorkersRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListArchivedWorkersRequest) JobClusterProtoAdapter(io.mantisrx.master.api.akka.route.proto.JobClusterProtoAdapter) Unmarshaller(akka.http.javadsl.unmarshalling.Unmarshaller) Metrics(io.mantisrx.common.metrics.Metrics) ResubmitWorkerRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ResubmitWorkerRequest) PostJobStatusRequest(io.mantisrx.server.core.PostJobStatusRequest) Jackson(io.mantisrx.master.api.akka.route.Jackson) BaseResponse(io.mantisrx.master.jobcluster.proto.BaseResponse) ScaleStageRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ScaleStageRequest) HttpMethods(akka.http.javadsl.model.HttpMethods) Collectors(java.util.stream.Collectors) ExceptionHandler(akka.http.javadsl.server.ExceptionHandler) List(java.util.List) JobRouteUtils.createListJobIdsRequest(io.mantisrx.master.api.akka.route.utils.JobRouteUtils.createListJobIdsRequest) CachingDirectives.alwaysCache(akka.http.javadsl.server.directives.CachingDirectives.alwaysCache) ActorSystem(akka.actor.ActorSystem) ConfigurationProvider(io.mantisrx.server.master.config.ConfigurationProvider) Optional(java.util.Optional) DataFormatAdapter(io.mantisrx.server.master.domain.DataFormatAdapter) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) JobRouteUtils.createListJobsRequest(io.mantisrx.master.api.akka.route.utils.JobRouteUtils.createListJobsRequest) Route(akka.http.javadsl.server.Route) MetricsRegistry(io.mantisrx.common.metrics.MetricsRegistry) StatusCodes(akka.http.javadsl.model.StatusCodes) Function(java.util.function.Function) PathMatchers(akka.http.javadsl.server.PathMatchers) WorkerEvent(io.mantisrx.server.master.scheduler.WorkerEvent) KillJobRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.KillJobRequest) Counter(io.mantisrx.common.metrics.Counter) Logger(org.slf4j.Logger) RouteResult(akka.http.javadsl.server.RouteResult) HttpHeader(akka.http.javadsl.model.HttpHeader) HttpRequest(akka.http.javadsl.model.HttpRequest) IOException(java.io.IOException) DEFAULT_LIST_ARCHIVED_WORKERS_LIMIT(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListArchivedWorkersRequest.DEFAULT_LIST_ARCHIVED_WORKERS_LIMIT) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) JobRouteUtils.createWorkerStatusRequest(io.mantisrx.master.api.akka.route.utils.JobRouteUtils.createWorkerStatusRequest) Collections(java.util.Collections) WorkerHeartbeat(io.mantisrx.master.jobcluster.job.worker.WorkerHeartbeat) Optional(java.util.Optional) PostJobStatusRequest(io.mantisrx.server.core.PostJobStatusRequest) IOException(java.io.IOException) ScaleStageRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ScaleStageRequest) WorkerHeartbeat(io.mantisrx.master.jobcluster.job.worker.WorkerHeartbeat) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) WorkerEvent(io.mantisrx.server.master.scheduler.WorkerEvent) ResubmitWorkerRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ResubmitWorkerRequest) KillJobRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.KillJobRequest) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) ListArchivedWorkersRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListArchivedWorkersRequest) JobId(io.mantisrx.server.master.domain.JobId) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto)

Example 5 with MantisWorkerMetadataWritable

use of io.mantisrx.server.master.store.MantisWorkerMetadataWritable in project mantis by Netflix.

the class JobRouteTest method testJobClusterGetJobDetail.

@Test(dependsOnMethods = { "testJobClusterGetJobsList" })
public void testJobClusterGetJobDetail() throws InterruptedException {
    final CountDownLatch latch = new CountDownLatch(1);
    final CompletionStage<HttpResponse> responseFuture = http.singleRequest(HttpRequest.GET(jobAPIEndpoint("list/sine-function-1")));
    responseFuture.thenCompose(r -> processRespFut(r, Optional.of(200))).whenComplete((msg, t) -> {
        String responseMessage = getResponseMessage(msg, t);
        logger.info("got response---> {}", responseMessage);
        MantisJobMetadataView mjm = null;
        try {
            mjm = Jackson.fromJSON(responseMessage, MantisJobMetadataView.class);
        } catch (IOException e) {
            logger.error("failed to deser json {}", responseMessage, e);
            fail("job info deser failed");
        }
        logger.info("jobInfo---> {}", mjm);
        assertNotNull(mjm);
        assertEquals(mjm.getJobMetadata().getJobId(), "sine-function-1");
        assertEquals(mjm.getJobMetadata().getName(), "sine-function");
        assertTrue(mjm.getStageMetadataList().size() > 0);
        MantisStageMetadataWritable msm = mjm.getStageMetadataList().get(0);
        assertEquals(1, msm.getNumWorkers());
        assertTrue(mjm.getWorkerMetadataList().size() > 0);
        MantisWorkerMetadataWritable mwm = mjm.getWorkerMetadataList().get(0);
        assertEquals("sine-function-1", mwm.getJobId());
        assertEquals(false, mwm.getCluster().isPresent());
        latch.countDown();
    });
    assertTrue(latch.await(2, TimeUnit.SECONDS));
}
Also used : JobStatusStreamRoute(io.mantisrx.master.api.akka.route.v1.JobStatusStreamRoute) TestHelpers(com.netflix.mantis.master.scheduler.TestHelpers) MantisJobDurationType(io.mantisrx.runtime.MantisJobDurationType) MantisJobState(io.mantisrx.runtime.MantisJobState) MasterDescription(io.mantisrx.server.core.master.MasterDescription) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) TypeReference(io.mantisrx.shaded.com.fasterxml.jackson.core.type.TypeReference) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) Test(org.testng.annotations.Test) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) JobsRoute(io.mantisrx.master.api.akka.route.v1.JobsRoute) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) JobClustersManagerActor(io.mantisrx.master.JobClustersManagerActor) JobPayloads(io.mantisrx.master.api.akka.payloads.JobPayloads) ActorMaterializer(akka.stream.ActorMaterializer) ActorRef(akka.actor.ActorRef) MantisMasterRoute(io.mantisrx.master.api.akka.route.MantisMasterRoute) Duration(java.time.Duration) Map(java.util.Map) JobClusterProtoAdapter(io.mantisrx.master.api.akka.route.proto.JobClusterProtoAdapter) Assert.fail(org.junit.Assert.fail) LastSubmittedJobIdStreamRoute(io.mantisrx.master.api.akka.route.v1.LastSubmittedJobIdStreamRoute) StatusEventSubscriberLoggingImpl(io.mantisrx.master.events.StatusEventSubscriberLoggingImpl) ServerBinding(akka.http.javadsl.ServerBinding) HttpCharsets(akka.http.javadsl.model.HttpCharsets) JobStatusRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobStatusRouteHandler) WorkerEventSubscriberLoggingImpl(io.mantisrx.master.events.WorkerEventSubscriberLoggingImpl) Jackson(io.mantisrx.master.api.akka.route.Jackson) BeforeClass(org.testng.annotations.BeforeClass) AdminMasterRoute(io.mantisrx.master.api.akka.route.v1.AdminMasterRoute) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) HttpMethods(akka.http.javadsl.model.HttpMethods) WorkerAssignments(io.mantisrx.server.core.WorkerAssignments) JobClusterRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) AgentClusterOperations(io.mantisrx.master.vm.AgentClusterOperations) CompletionStage(java.util.concurrent.CompletionStage) NotUsed(akka.NotUsed) AgentClustersRoute(io.mantisrx.master.api.akka.route.v1.AgentClustersRoute) ActorSystem(akka.actor.ActorSystem) JobDiscoveryStreamRoute(io.mantisrx.master.api.akka.route.v1.JobDiscoveryStreamRoute) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) Optional(java.util.Optional) Mockito.mock(org.mockito.Mockito.mock) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) Flow(akka.stream.javadsl.Flow) JobRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobRouteHandlerAkkaImpl) LeaderRedirectionFilter(io.mantisrx.server.master.LeaderRedirectionFilter) Matchers.anyString(org.mockito.Matchers.anyString) AuditEventSubscriberLoggingImpl(io.mantisrx.master.events.AuditEventSubscriberLoggingImpl) Observable(rx.Observable) JobClusterPayloads(io.mantisrx.master.api.akka.payloads.JobClusterPayloads) CompactJobInfo(io.mantisrx.server.master.http.api.CompactJobInfo) LifecycleEventPublisherImpl(io.mantisrx.master.events.LifecycleEventPublisherImpl) JobTestHelper(io.mantisrx.master.jobcluster.job.JobTestHelper) ByteString(akka.util.ByteString) HttpEntity(akka.http.javadsl.model.HttpEntity) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) LocalMasterMonitor(io.mantisrx.server.core.master.LocalMasterMonitor) LeadershipManagerLocalImpl(io.mantisrx.server.master.LeadershipManagerLocalImpl) JobClusterRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandlerAkkaImpl) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) AfterClass(org.testng.annotations.AfterClass) Logger(org.slf4j.Logger) Assert.assertNotNull(org.junit.Assert.assertNotNull) Http(akka.http.javadsl.Http) HttpRequest(akka.http.javadsl.model.HttpRequest) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) ContentTypes(akka.http.javadsl.model.ContentTypes) HttpEntities(akka.http.javadsl.model.HttpEntities) HttpResponse(akka.http.javadsl.model.HttpResponse) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) ConnectHttp(akka.http.javadsl.ConnectHttp) MediaTypes(akka.http.javadsl.model.MediaTypes) JobClustersRoute(io.mantisrx.master.api.akka.route.v1.JobClustersRoute) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) HttpResponse(akka.http.javadsl.model.HttpResponse) Matchers.anyString(org.mockito.Matchers.anyString) ByteString(akka.util.ByteString) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Aggregations

MantisWorkerMetadataWritable (io.mantisrx.server.master.store.MantisWorkerMetadataWritable)9 ActorSystem (akka.actor.ActorSystem)4 HttpMethods (akka.http.javadsl.model.HttpMethods)4 HttpRequest (akka.http.javadsl.model.HttpRequest)4 Jackson (io.mantisrx.master.api.akka.route.Jackson)4 JobRouteHandler (io.mantisrx.master.api.akka.route.handlers.JobRouteHandler)4 JobClusterProtoAdapter (io.mantisrx.master.api.akka.route.proto.JobClusterProtoAdapter)4 NotUsed (akka.NotUsed)3 ActorRef (akka.actor.ActorRef)3 ConnectHttp (akka.http.javadsl.ConnectHttp)3 Http (akka.http.javadsl.Http)3 ServerBinding (akka.http.javadsl.ServerBinding)3 ContentTypes (akka.http.javadsl.model.ContentTypes)3 HttpCharsets (akka.http.javadsl.model.HttpCharsets)3 HttpEntities (akka.http.javadsl.model.HttpEntities)3 HttpEntity (akka.http.javadsl.model.HttpEntity)3 HttpResponse (akka.http.javadsl.model.HttpResponse)3 MediaTypes (akka.http.javadsl.model.MediaTypes)3 MantisJobMetadataView (io.mantisrx.master.jobcluster.job.MantisJobMetadataView)3 IMantisWorkerMetadata (io.mantisrx.master.jobcluster.job.worker.IMantisWorkerMetadata)3