Search in sources :

Example 1 with MantisStageMetadataWritable

use of io.mantisrx.server.master.store.MantisStageMetadataWritable in project mantis by Netflix.

the class JobRouteTest method testJobClusterGetJobsList.

@Test(dependsOnMethods = { "testJobClusterGetJobIds" })
public void testJobClusterGetJobsList() throws InterruptedException {
    final CountDownLatch latch = new CountDownLatch(1);
    final CompletionStage<HttpResponse> responseFuture = http.singleRequest(HttpRequest.GET(jobAPIEndpoint("list")));
    responseFuture.thenCompose(r -> processRespFut(r, Optional.of(200))).whenComplete((msg, t) -> {
        String responseMessage = getResponseMessage(msg, t);
        logger.info("got response---> {}", responseMessage);
        List<MantisJobMetadataView> jobInfos = Collections.emptyList();
        try {
            jobInfos = Jackson.fromJSON(responseMessage, new TypeReference<List<MantisJobMetadataView>>() {
            });
        } catch (IOException e) {
            logger.error("failed to deser json {}", responseMessage, e);
            fail("job list deser failed");
        }
        logger.info("jobInfos---> {}", jobInfos);
        assertEquals(1, jobInfos.size());
        MantisJobMetadataView mjm = jobInfos.get(0);
        assertEquals(mjm.getJobMetadata().getJobId(), "sine-function-1");
        assertEquals(mjm.getJobMetadata().getName(), "sine-function");
        assertTrue(mjm.getStageMetadataList().size() > 0);
        MantisStageMetadataWritable msm = mjm.getStageMetadataList().get(0);
        assertEquals(1, msm.getNumWorkers());
        assertTrue(mjm.getWorkerMetadataList().size() > 0);
        MantisWorkerMetadataWritable mwm = mjm.getWorkerMetadataList().get(0);
        assertEquals("sine-function-1", mwm.getJobId());
        assertEquals(false, mwm.getCluster().isPresent());
        latch.countDown();
    });
    assertTrue(latch.await(2, TimeUnit.SECONDS));
}
Also used : JobStatusStreamRoute(io.mantisrx.master.api.akka.route.v1.JobStatusStreamRoute) TestHelpers(com.netflix.mantis.master.scheduler.TestHelpers) MantisJobDurationType(io.mantisrx.runtime.MantisJobDurationType) MantisJobState(io.mantisrx.runtime.MantisJobState) MasterDescription(io.mantisrx.server.core.master.MasterDescription) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) TypeReference(io.mantisrx.shaded.com.fasterxml.jackson.core.type.TypeReference) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) Test(org.testng.annotations.Test) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) JobsRoute(io.mantisrx.master.api.akka.route.v1.JobsRoute) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) JobClustersManagerActor(io.mantisrx.master.JobClustersManagerActor) JobPayloads(io.mantisrx.master.api.akka.payloads.JobPayloads) ActorMaterializer(akka.stream.ActorMaterializer) ActorRef(akka.actor.ActorRef) MantisMasterRoute(io.mantisrx.master.api.akka.route.MantisMasterRoute) Duration(java.time.Duration) Map(java.util.Map) JobClusterProtoAdapter(io.mantisrx.master.api.akka.route.proto.JobClusterProtoAdapter) Assert.fail(org.junit.Assert.fail) LastSubmittedJobIdStreamRoute(io.mantisrx.master.api.akka.route.v1.LastSubmittedJobIdStreamRoute) StatusEventSubscriberLoggingImpl(io.mantisrx.master.events.StatusEventSubscriberLoggingImpl) ServerBinding(akka.http.javadsl.ServerBinding) HttpCharsets(akka.http.javadsl.model.HttpCharsets) JobStatusRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobStatusRouteHandler) WorkerEventSubscriberLoggingImpl(io.mantisrx.master.events.WorkerEventSubscriberLoggingImpl) Jackson(io.mantisrx.master.api.akka.route.Jackson) BeforeClass(org.testng.annotations.BeforeClass) AdminMasterRoute(io.mantisrx.master.api.akka.route.v1.AdminMasterRoute) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) HttpMethods(akka.http.javadsl.model.HttpMethods) WorkerAssignments(io.mantisrx.server.core.WorkerAssignments) JobClusterRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) AgentClusterOperations(io.mantisrx.master.vm.AgentClusterOperations) CompletionStage(java.util.concurrent.CompletionStage) NotUsed(akka.NotUsed) AgentClustersRoute(io.mantisrx.master.api.akka.route.v1.AgentClustersRoute) ActorSystem(akka.actor.ActorSystem) JobDiscoveryStreamRoute(io.mantisrx.master.api.akka.route.v1.JobDiscoveryStreamRoute) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) Optional(java.util.Optional) Mockito.mock(org.mockito.Mockito.mock) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) Flow(akka.stream.javadsl.Flow) JobRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobRouteHandlerAkkaImpl) LeaderRedirectionFilter(io.mantisrx.server.master.LeaderRedirectionFilter) Matchers.anyString(org.mockito.Matchers.anyString) AuditEventSubscriberLoggingImpl(io.mantisrx.master.events.AuditEventSubscriberLoggingImpl) Observable(rx.Observable) JobClusterPayloads(io.mantisrx.master.api.akka.payloads.JobClusterPayloads) CompactJobInfo(io.mantisrx.server.master.http.api.CompactJobInfo) LifecycleEventPublisherImpl(io.mantisrx.master.events.LifecycleEventPublisherImpl) JobTestHelper(io.mantisrx.master.jobcluster.job.JobTestHelper) ByteString(akka.util.ByteString) HttpEntity(akka.http.javadsl.model.HttpEntity) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) LocalMasterMonitor(io.mantisrx.server.core.master.LocalMasterMonitor) LeadershipManagerLocalImpl(io.mantisrx.server.master.LeadershipManagerLocalImpl) JobClusterRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandlerAkkaImpl) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) AfterClass(org.testng.annotations.AfterClass) Logger(org.slf4j.Logger) Assert.assertNotNull(org.junit.Assert.assertNotNull) Http(akka.http.javadsl.Http) HttpRequest(akka.http.javadsl.model.HttpRequest) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) ContentTypes(akka.http.javadsl.model.ContentTypes) HttpEntities(akka.http.javadsl.model.HttpEntities) HttpResponse(akka.http.javadsl.model.HttpResponse) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) ConnectHttp(akka.http.javadsl.ConnectHttp) MediaTypes(akka.http.javadsl.model.MediaTypes) JobClustersRoute(io.mantisrx.master.api.akka.route.v1.JobClustersRoute) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) HttpResponse(akka.http.javadsl.model.HttpResponse) Matchers.anyString(org.mockito.Matchers.anyString) ByteString(akka.util.ByteString) TypeReference(io.mantisrx.shaded.com.fasterxml.jackson.core.type.TypeReference) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Example 2 with MantisStageMetadataWritable

use of io.mantisrx.server.master.store.MantisStageMetadataWritable in project mantis by Netflix.

the class JobRouteTest method testJobClusterGetJobDetail.

@Test(dependsOnMethods = { "testJobClusterGetJobsList" })
public void testJobClusterGetJobDetail() throws InterruptedException {
    final CountDownLatch latch = new CountDownLatch(1);
    final CompletionStage<HttpResponse> responseFuture = http.singleRequest(HttpRequest.GET(jobAPIEndpoint("list/sine-function-1")));
    responseFuture.thenCompose(r -> processRespFut(r, Optional.of(200))).whenComplete((msg, t) -> {
        String responseMessage = getResponseMessage(msg, t);
        logger.info("got response---> {}", responseMessage);
        MantisJobMetadataView mjm = null;
        try {
            mjm = Jackson.fromJSON(responseMessage, MantisJobMetadataView.class);
        } catch (IOException e) {
            logger.error("failed to deser json {}", responseMessage, e);
            fail("job info deser failed");
        }
        logger.info("jobInfo---> {}", mjm);
        assertNotNull(mjm);
        assertEquals(mjm.getJobMetadata().getJobId(), "sine-function-1");
        assertEquals(mjm.getJobMetadata().getName(), "sine-function");
        assertTrue(mjm.getStageMetadataList().size() > 0);
        MantisStageMetadataWritable msm = mjm.getStageMetadataList().get(0);
        assertEquals(1, msm.getNumWorkers());
        assertTrue(mjm.getWorkerMetadataList().size() > 0);
        MantisWorkerMetadataWritable mwm = mjm.getWorkerMetadataList().get(0);
        assertEquals("sine-function-1", mwm.getJobId());
        assertEquals(false, mwm.getCluster().isPresent());
        latch.countDown();
    });
    assertTrue(latch.await(2, TimeUnit.SECONDS));
}
Also used : JobStatusStreamRoute(io.mantisrx.master.api.akka.route.v1.JobStatusStreamRoute) TestHelpers(com.netflix.mantis.master.scheduler.TestHelpers) MantisJobDurationType(io.mantisrx.runtime.MantisJobDurationType) MantisJobState(io.mantisrx.runtime.MantisJobState) MasterDescription(io.mantisrx.server.core.master.MasterDescription) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) TypeReference(io.mantisrx.shaded.com.fasterxml.jackson.core.type.TypeReference) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) Test(org.testng.annotations.Test) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) JobsRoute(io.mantisrx.master.api.akka.route.v1.JobsRoute) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) JobClustersManagerActor(io.mantisrx.master.JobClustersManagerActor) JobPayloads(io.mantisrx.master.api.akka.payloads.JobPayloads) ActorMaterializer(akka.stream.ActorMaterializer) ActorRef(akka.actor.ActorRef) MantisMasterRoute(io.mantisrx.master.api.akka.route.MantisMasterRoute) Duration(java.time.Duration) Map(java.util.Map) JobClusterProtoAdapter(io.mantisrx.master.api.akka.route.proto.JobClusterProtoAdapter) Assert.fail(org.junit.Assert.fail) LastSubmittedJobIdStreamRoute(io.mantisrx.master.api.akka.route.v1.LastSubmittedJobIdStreamRoute) StatusEventSubscriberLoggingImpl(io.mantisrx.master.events.StatusEventSubscriberLoggingImpl) ServerBinding(akka.http.javadsl.ServerBinding) HttpCharsets(akka.http.javadsl.model.HttpCharsets) JobStatusRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobStatusRouteHandler) WorkerEventSubscriberLoggingImpl(io.mantisrx.master.events.WorkerEventSubscriberLoggingImpl) Jackson(io.mantisrx.master.api.akka.route.Jackson) BeforeClass(org.testng.annotations.BeforeClass) AdminMasterRoute(io.mantisrx.master.api.akka.route.v1.AdminMasterRoute) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) HttpMethods(akka.http.javadsl.model.HttpMethods) WorkerAssignments(io.mantisrx.server.core.WorkerAssignments) JobClusterRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) AgentClusterOperations(io.mantisrx.master.vm.AgentClusterOperations) CompletionStage(java.util.concurrent.CompletionStage) NotUsed(akka.NotUsed) AgentClustersRoute(io.mantisrx.master.api.akka.route.v1.AgentClustersRoute) ActorSystem(akka.actor.ActorSystem) JobDiscoveryStreamRoute(io.mantisrx.master.api.akka.route.v1.JobDiscoveryStreamRoute) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) Optional(java.util.Optional) Mockito.mock(org.mockito.Mockito.mock) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) Flow(akka.stream.javadsl.Flow) JobRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobRouteHandlerAkkaImpl) LeaderRedirectionFilter(io.mantisrx.server.master.LeaderRedirectionFilter) Matchers.anyString(org.mockito.Matchers.anyString) AuditEventSubscriberLoggingImpl(io.mantisrx.master.events.AuditEventSubscriberLoggingImpl) Observable(rx.Observable) JobClusterPayloads(io.mantisrx.master.api.akka.payloads.JobClusterPayloads) CompactJobInfo(io.mantisrx.server.master.http.api.CompactJobInfo) LifecycleEventPublisherImpl(io.mantisrx.master.events.LifecycleEventPublisherImpl) JobTestHelper(io.mantisrx.master.jobcluster.job.JobTestHelper) ByteString(akka.util.ByteString) HttpEntity(akka.http.javadsl.model.HttpEntity) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) LocalMasterMonitor(io.mantisrx.server.core.master.LocalMasterMonitor) LeadershipManagerLocalImpl(io.mantisrx.server.master.LeadershipManagerLocalImpl) JobClusterRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandlerAkkaImpl) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) AfterClass(org.testng.annotations.AfterClass) Logger(org.slf4j.Logger) Assert.assertNotNull(org.junit.Assert.assertNotNull) Http(akka.http.javadsl.Http) HttpRequest(akka.http.javadsl.model.HttpRequest) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) ContentTypes(akka.http.javadsl.model.ContentTypes) HttpEntities(akka.http.javadsl.model.HttpEntities) HttpResponse(akka.http.javadsl.model.HttpResponse) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) ConnectHttp(akka.http.javadsl.ConnectHttp) MediaTypes(akka.http.javadsl.model.MediaTypes) JobClustersRoute(io.mantisrx.master.api.akka.route.v1.JobClustersRoute) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) HttpResponse(akka.http.javadsl.model.HttpResponse) Matchers.anyString(org.mockito.Matchers.anyString) ByteString(akka.util.ByteString) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Example 3 with MantisStageMetadataWritable

use of io.mantisrx.server.master.store.MantisStageMetadataWritable in project mantis by Netflix.

the class DataFormatAdapterTest method convertMantisStageMetaTest.

@Test
public void convertMantisStageMetaTest() {
    Map<StageScalingPolicy.ScalingReason, StageScalingPolicy.Strategy> smap = new HashMap<>();
    smap.put(StageScalingPolicy.ScalingReason.CPU, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.CPU, 0.5, 0.75, null));
    smap.put(StageScalingPolicy.ScalingReason.DataDrop, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.DataDrop, 0.0, 2.0, null));
    int stageNo = 1;
    int min = 3;
    int max = 10;
    int increment = 1;
    int decrement = 1;
    int coolDownSecs = 300;
    StageScalingPolicy stageScalingPolicy = new StageScalingPolicy(stageNo, min, max, increment, decrement, coolDownSecs, smap);
    List<JobConstraints> softConstraintsList = new ArrayList<>();
    softConstraintsList.add(JobConstraints.ExclusiveHost);
    List<JobConstraints> hardConstraintsList = new ArrayList<>();
    hardConstraintsList.add(JobConstraints.M3Cluster);
    JobId jobId = new JobId("cName", 1);
    int numWorkers = 1;
    int numStages = 2;
    boolean isScalable = true;
    IMantisStageMetadata stageMeta = new MantisStageMetadataImpl.Builder().withStageNum(stageNo).withScalingPolicy(stageScalingPolicy).withNumWorkers(numWorkers).withMachineDefinition(DEFAULT_MACHINE_DEFINITION).withNumStages(numStages).withSoftConstraints(softConstraintsList).withHardConstraints(hardConstraintsList).withJobId(jobId).isScalable(isScalable).build();
    MantisStageMetadataWritable stageMetadataWritable = DataFormatAdapter.convertMantisStageMetadataToMantisStageMetadataWriteable(stageMeta);
    assertEquals(jobId.getId(), stageMetadataWritable.getJobId());
    assertEquals(JobConstraints.M3Cluster, stageMetadataWritable.getHardConstraints().get(0));
    assertEquals(JobConstraints.ExclusiveHost, stageMetadataWritable.getSoftConstraints().get(0));
    assertEquals(stageScalingPolicy, stageMetadataWritable.getScalingPolicy());
    assertTrue(stageMetadataWritable.getScalable());
    assertEquals(DEFAULT_MACHINE_DEFINITION, stageMetadataWritable.getMachineDefinition());
    assertEquals(numWorkers, stageMetadataWritable.getNumWorkers());
    assertEquals(numStages, stageMetadataWritable.getNumStages());
    assertEquals(stageNo, stageMetadataWritable.getStageNum());
    IMantisStageMetadata reconverted = DataFormatAdapter.convertMantisStageMetadataWriteableToMantisStageMetadata(stageMetadataWritable, eventPublisher);
    assertEquals(stageMeta, reconverted);
}
Also used : HashMap(java.util.HashMap) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) ArrayList(java.util.ArrayList) JobConstraints(io.mantisrx.runtime.JobConstraints) StageScalingPolicy(io.mantisrx.runtime.descriptor.StageScalingPolicy) IMantisStageMetadata(io.mantisrx.master.jobcluster.job.IMantisStageMetadata) Test(org.junit.Test)

Aggregations

NotUsed (akka.NotUsed)2 ActorRef (akka.actor.ActorRef)2 ActorSystem (akka.actor.ActorSystem)2 ConnectHttp (akka.http.javadsl.ConnectHttp)2 Http (akka.http.javadsl.Http)2 ServerBinding (akka.http.javadsl.ServerBinding)2 ContentTypes (akka.http.javadsl.model.ContentTypes)2 HttpCharsets (akka.http.javadsl.model.HttpCharsets)2 HttpEntities (akka.http.javadsl.model.HttpEntities)2 HttpEntity (akka.http.javadsl.model.HttpEntity)2 HttpMethods (akka.http.javadsl.model.HttpMethods)2 HttpRequest (akka.http.javadsl.model.HttpRequest)2 HttpResponse (akka.http.javadsl.model.HttpResponse)2 MediaTypes (akka.http.javadsl.model.MediaTypes)2 ActorMaterializer (akka.stream.ActorMaterializer)2 Flow (akka.stream.javadsl.Flow)2 ByteString (akka.util.ByteString)2 TestHelpers (com.netflix.mantis.master.scheduler.TestHelpers)2 JobClustersManagerActor (io.mantisrx.master.JobClustersManagerActor)2 JobClusterPayloads (io.mantisrx.master.api.akka.payloads.JobClusterPayloads)2