Search in sources :

Example 1 with NamedJobInfo

use of io.mantisrx.server.core.NamedJobInfo in project mantis by Netflix.

the class JobDiscoveryStreamRouteTest method testLastSubmittedJobIdStreamForNonExistentJob.

@Test
public void testLastSubmittedJobIdStreamForNonExistentJob() throws InterruptedException {
    // The current behavior of Mantis client is to retry non-200 responses
    // This test overrides the default retry/repeat behavior to test a namedjob info observable would complete if the job cluster requested is non-existent
    final CountDownLatch latch = new CountDownLatch(1);
    Observable<NamedJobInfo> jobSchedulingInfoObservable = mantisClient.namedJobInfo("testJobCluster", obs -> Observable.just(1), obs -> Observable.empty());
    jobSchedulingInfoObservable.doOnNext(x -> logger.info("onNext {}", x)).doOnError(t -> logger.warn("onError", t)).doOnCompleted(() -> {
        logger.info("onCompleted");
        latch.countDown();
    }).subscribe();
    latch.await();
}
Also used : AgentsErrorMonitorActor(io.mantisrx.master.scheduler.AgentsErrorMonitorActor) TestHelpers(com.netflix.mantis.master.scheduler.TestHelpers) Flow(akka.stream.javadsl.Flow) LoggerFactory(org.slf4j.LoggerFactory) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) Test(org.testng.annotations.Test) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) Function(java.util.function.Function) AuditEventSubscriberLoggingImpl(io.mantisrx.master.events.AuditEventSubscriberLoggingImpl) JobClustersManagerActor(io.mantisrx.master.JobClustersManagerActor) Observable(rx.Observable) ActorMaterializer(akka.stream.ActorMaterializer) LifecycleEventPublisherImpl(io.mantisrx.master.events.LifecycleEventPublisherImpl) JobTestHelper(io.mantisrx.master.jobcluster.job.JobTestHelper) ActorRef(akka.actor.ActorRef) Duration(java.time.Duration) StatusEventSubscriberLoggingImpl(io.mantisrx.master.events.StatusEventSubscriberLoggingImpl) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) ServerBinding(akka.http.javadsl.ServerBinding) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) AfterClass(org.testng.annotations.AfterClass) Logger(org.slf4j.Logger) WorkerEventSubscriberLoggingImpl(io.mantisrx.master.events.WorkerEventSubscriberLoggingImpl) Http(akka.http.javadsl.Http) HttpRequest(akka.http.javadsl.model.HttpRequest) BeforeClass(org.testng.annotations.BeforeClass) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) HttpResponse(akka.http.javadsl.model.HttpResponse) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) CountDownLatch(java.util.concurrent.CountDownLatch) ConnectHttp(akka.http.javadsl.ConnectHttp) CompletionStage(java.util.concurrent.CompletionStage) NotUsed(akka.NotUsed) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Example 2 with NamedJobInfo

use of io.mantisrx.server.core.NamedJobInfo in project mantis by Netflix.

the class SourceJobWorkerMetricsSubscriptionTest method testGetResultsForAllSourceJobs.

@Test
public void testGetResultsForAllSourceJobs() throws Exception {
    List<SourceJobParameters.TargetInfo> infos = ImmutableList.of(new SourceJobParameters.TargetInfoBuilder().withSourceJobName("jobA").withQuery("criterion").withClientId("client1").build(), new SourceJobParameters.TargetInfoBuilder().withSourceJobName("jobA").withQuery("criterion").withClientId("client2").build(), new SourceJobParameters.TargetInfoBuilder().withSourceJobName("jobB").withQuery("criterion").withClientId("client1").build(), new SourceJobParameters.TargetInfoBuilder().withSourceJobName("jobB").withQuery("criterion").withClientId("client3").build());
    MantisMasterClientApi masterClient = mock(MantisMasterClientApi.class);
    SourceJobWorkerMetricsSubscription sub = spy(new SourceJobWorkerMetricsSubscription(infos, masterClient, null, new AutoScaleMetricsConfig()));
    when(masterClient.namedJobInfo("jobA")).thenReturn(Observable.just(new NamedJobInfo("jobA", "jobA-1")));
    when(masterClient.namedJobInfo("jobB")).thenReturn(Observable.just(new NamedJobInfo("jobA", "jobB-2")));
    doReturn(Observable.just(Observable.just(new MantisServerSentEvent("jobA-event")))).when(sub).getResultsForJobId(eq("jobA-1"), any());
    doReturn(Observable.just(Observable.just(new MantisServerSentEvent("jobB-event")))).when(sub).getResultsForJobId(eq("jobB-2"), any());
    CountDownLatch latch = new CountDownLatch(2);
    Observable.merge(sub.getResults()).doOnNext(event -> {
        if ("jobA-event".equals(event.getEventAsString()) || "jobB-event".equals(event.getEventAsString())) {
            latch.countDown();
        }
    }).subscribe();
    latch.await(10, TimeUnit.SECONDS);
    assertEquals(0, latch.getCount());
    Set<String> jobAMetrics = ImmutableSet.of("PushServerSse:clientId=client1:*", "PushServerSse:clientId=client2:*", "ServerSentEventRequestHandler:clientId=client1:*", "ServerSentEventRequestHandler:clientId=client2:*");
    verify(sub, times(1)).getResultsForJobId("jobA-1", jobAMetrics);
    jobAMetrics = ImmutableSet.of("PushServerSse:clientId=client1:*", "PushServerSse:clientId=client3:*", "ServerSentEventRequestHandler:clientId=client1:*", "ServerSentEventRequestHandler:clientId=client3:*");
    verify(sub, times(1)).getResultsForJobId("jobB-2", jobAMetrics);
}
Also used : ImmutableSet(io.mantisrx.shaded.com.google.common.collect.ImmutableSet) SourceJobParameters(io.mantisrx.runtime.parameter.SourceJobParameters) MantisMasterClientApi(io.mantisrx.server.master.client.MantisMasterClientApi) Set(java.util.Set) Test(org.junit.Test) ImmutableMap(io.mantisrx.shaded.com.google.common.collect.ImmutableMap) Observable(rx.Observable) TimeUnit(java.util.concurrent.TimeUnit) Mockito(org.mockito.Mockito) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) MantisServerSentEvent(io.mantisrx.common.MantisServerSentEvent) Map(java.util.Map) Assert.assertEquals(org.junit.Assert.assertEquals) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) ImmutableList(io.mantisrx.shaded.com.google.common.collect.ImmutableList) SourceJobParameters(io.mantisrx.runtime.parameter.SourceJobParameters) CountDownLatch(java.util.concurrent.CountDownLatch) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) MantisServerSentEvent(io.mantisrx.common.MantisServerSentEvent) MantisMasterClientApi(io.mantisrx.server.master.client.MantisMasterClientApi) Test(org.junit.Test)

Example 3 with NamedJobInfo

use of io.mantisrx.server.core.NamedJobInfo in project mantis by Netflix.

the class JobDiscoveryRouteTest method testNamedJobInfoStreamForNonExistentJob.

@Test
public void testNamedJobInfoStreamForNonExistentJob() throws InterruptedException {
    // The current behavior of Mantis client is to retry non-200 responses
    // This test overrides the default retry/repeat behavior to test a namedjob info observable would complete if the job cluster requested is non-existent
    final CountDownLatch latch = new CountDownLatch(1);
    Observable<NamedJobInfo> jobSchedulingInfoObservable = mantisClient.namedJobInfo("testJobCluster", obs -> Observable.just(1), obs -> Observable.empty());
    jobSchedulingInfoObservable.doOnNext(x -> logger.info("onNext {}", x)).doOnError(t -> logger.warn("onError", t)).doOnCompleted(() -> {
        logger.info("onCompleted");
        latch.countDown();
    }).subscribe();
    latch.await();
}
Also used : AgentsErrorMonitorActor(io.mantisrx.master.scheduler.AgentsErrorMonitorActor) TestHelpers(com.netflix.mantis.master.scheduler.TestHelpers) Flow(akka.stream.javadsl.Flow) LoggerFactory(org.slf4j.LoggerFactory) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) Test(org.testng.annotations.Test) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) Function(java.util.function.Function) AuditEventSubscriberLoggingImpl(io.mantisrx.master.events.AuditEventSubscriberLoggingImpl) JobClustersManagerActor(io.mantisrx.master.JobClustersManagerActor) Observable(rx.Observable) ActorMaterializer(akka.stream.ActorMaterializer) LifecycleEventPublisherImpl(io.mantisrx.master.events.LifecycleEventPublisherImpl) JobTestHelper(io.mantisrx.master.jobcluster.job.JobTestHelper) ActorRef(akka.actor.ActorRef) Duration(java.time.Duration) StatusEventSubscriberLoggingImpl(io.mantisrx.master.events.StatusEventSubscriberLoggingImpl) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) ServerBinding(akka.http.javadsl.ServerBinding) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) AfterClass(org.testng.annotations.AfterClass) Logger(org.slf4j.Logger) WorkerEventSubscriberLoggingImpl(io.mantisrx.master.events.WorkerEventSubscriberLoggingImpl) Http(akka.http.javadsl.Http) HttpRequest(akka.http.javadsl.model.HttpRequest) BeforeClass(org.testng.annotations.BeforeClass) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) HttpResponse(akka.http.javadsl.model.HttpResponse) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) CountDownLatch(java.util.concurrent.CountDownLatch) ConnectHttp(akka.http.javadsl.ConnectHttp) CompletionStage(java.util.concurrent.CompletionStage) NotUsed(akka.NotUsed) ActorSystem(akka.actor.ActorSystem) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Example 4 with NamedJobInfo

use of io.mantisrx.server.core.NamedJobInfo in project mantis by Netflix.

the class JobRouteTest method testNamedJobInfoStream.

@Test(dependsOnMethods = { "testJobClusterGetJobsCompact" })
public void testNamedJobInfoStream() throws InterruptedException {
    final CountDownLatch latch = new CountDownLatch(1);
    final String jobCluster = "sine-function";
    Observable<NamedJobInfo> namedJobInfo = mantisClient.namedJobInfo(jobCluster);
    namedJobInfo.doOnNext(lastSubmittedJobId -> {
        logger.info("namedJobInfo {} {}", lastSubmittedJobId.getName(), lastSubmittedJobId.getJobId());
        try {
            lastSubmittedJobId.getName();
            assertEquals("sine-function", lastSubmittedJobId.getName());
            assertEquals("sine-function-1", lastSubmittedJobId.getJobId());
        } catch (Exception e) {
            logger.error("caught exception", e);
            org.testng.Assert.fail("testNamedJobInfoStream test failed with exception " + e.getMessage(), e);
        }
        latch.countDown();
    }).doOnError(t -> logger.warn("onError", t)).doOnCompleted(() -> logger.info("onCompleted")).doAfterTerminate(() -> latch.countDown()).subscribe();
    latch.await();
}
Also used : JobStatusStreamRoute(io.mantisrx.master.api.akka.route.v1.JobStatusStreamRoute) TestHelpers(com.netflix.mantis.master.scheduler.TestHelpers) MantisJobDurationType(io.mantisrx.runtime.MantisJobDurationType) MantisJobState(io.mantisrx.runtime.MantisJobState) MasterDescription(io.mantisrx.server.core.master.MasterDescription) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) TypeReference(io.mantisrx.shaded.com.fasterxml.jackson.core.type.TypeReference) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) Test(org.testng.annotations.Test) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) JobsRoute(io.mantisrx.master.api.akka.route.v1.JobsRoute) MantisStageMetadataWritable(io.mantisrx.server.master.store.MantisStageMetadataWritable) JobClustersManagerActor(io.mantisrx.master.JobClustersManagerActor) JobPayloads(io.mantisrx.master.api.akka.payloads.JobPayloads) ActorMaterializer(akka.stream.ActorMaterializer) ActorRef(akka.actor.ActorRef) MantisMasterRoute(io.mantisrx.master.api.akka.route.MantisMasterRoute) Duration(java.time.Duration) Map(java.util.Map) JobClusterProtoAdapter(io.mantisrx.master.api.akka.route.proto.JobClusterProtoAdapter) Assert.fail(org.junit.Assert.fail) LastSubmittedJobIdStreamRoute(io.mantisrx.master.api.akka.route.v1.LastSubmittedJobIdStreamRoute) StatusEventSubscriberLoggingImpl(io.mantisrx.master.events.StatusEventSubscriberLoggingImpl) ServerBinding(akka.http.javadsl.ServerBinding) HttpCharsets(akka.http.javadsl.model.HttpCharsets) JobStatusRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobStatusRouteHandler) WorkerEventSubscriberLoggingImpl(io.mantisrx.master.events.WorkerEventSubscriberLoggingImpl) Jackson(io.mantisrx.master.api.akka.route.Jackson) BeforeClass(org.testng.annotations.BeforeClass) AdminMasterRoute(io.mantisrx.master.api.akka.route.v1.AdminMasterRoute) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) HttpMethods(akka.http.javadsl.model.HttpMethods) WorkerAssignments(io.mantisrx.server.core.WorkerAssignments) JobClusterRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) AgentClusterOperations(io.mantisrx.master.vm.AgentClusterOperations) CompletionStage(java.util.concurrent.CompletionStage) NotUsed(akka.NotUsed) AgentClustersRoute(io.mantisrx.master.api.akka.route.v1.AgentClustersRoute) ActorSystem(akka.actor.ActorSystem) JobDiscoveryStreamRoute(io.mantisrx.master.api.akka.route.v1.JobDiscoveryStreamRoute) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) Optional(java.util.Optional) Mockito.mock(org.mockito.Mockito.mock) MantisJobMetadataView(io.mantisrx.master.jobcluster.job.MantisJobMetadataView) MantisWorkerMetadataWritable(io.mantisrx.server.master.store.MantisWorkerMetadataWritable) Flow(akka.stream.javadsl.Flow) JobRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobRouteHandlerAkkaImpl) LeaderRedirectionFilter(io.mantisrx.server.master.LeaderRedirectionFilter) Matchers.anyString(org.mockito.Matchers.anyString) AuditEventSubscriberLoggingImpl(io.mantisrx.master.events.AuditEventSubscriberLoggingImpl) Observable(rx.Observable) JobClusterPayloads(io.mantisrx.master.api.akka.payloads.JobClusterPayloads) CompactJobInfo(io.mantisrx.server.master.http.api.CompactJobInfo) LifecycleEventPublisherImpl(io.mantisrx.master.events.LifecycleEventPublisherImpl) JobTestHelper(io.mantisrx.master.jobcluster.job.JobTestHelper) ByteString(akka.util.ByteString) HttpEntity(akka.http.javadsl.model.HttpEntity) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) LocalMasterMonitor(io.mantisrx.server.core.master.LocalMasterMonitor) LeadershipManagerLocalImpl(io.mantisrx.server.master.LeadershipManagerLocalImpl) JobClusterRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandlerAkkaImpl) JobSchedulingInfo(io.mantisrx.server.core.JobSchedulingInfo) AfterClass(org.testng.annotations.AfterClass) Logger(org.slf4j.Logger) Assert.assertNotNull(org.junit.Assert.assertNotNull) Http(akka.http.javadsl.Http) HttpRequest(akka.http.javadsl.model.HttpRequest) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) ContentTypes(akka.http.javadsl.model.ContentTypes) HttpEntities(akka.http.javadsl.model.HttpEntities) HttpResponse(akka.http.javadsl.model.HttpResponse) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) ConnectHttp(akka.http.javadsl.ConnectHttp) MediaTypes(akka.http.javadsl.model.MediaTypes) JobClustersRoute(io.mantisrx.master.api.akka.route.v1.JobClustersRoute) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) NamedJobInfo(io.mantisrx.server.core.NamedJobInfo) Matchers.anyString(org.mockito.Matchers.anyString) ByteString(akka.util.ByteString) CountDownLatch(java.util.concurrent.CountDownLatch) TimeoutException(java.util.concurrent.TimeoutException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) Test(org.testng.annotations.Test)

Aggregations

NamedJobInfo (io.mantisrx.server.core.NamedJobInfo)4 CountDownLatch (java.util.concurrent.CountDownLatch)4 Observable (rx.Observable)4 NotUsed (akka.NotUsed)3 ActorRef (akka.actor.ActorRef)3 ConnectHttp (akka.http.javadsl.ConnectHttp)3 Http (akka.http.javadsl.Http)3 ServerBinding (akka.http.javadsl.ServerBinding)3 HttpRequest (akka.http.javadsl.model.HttpRequest)3 HttpResponse (akka.http.javadsl.model.HttpResponse)3 ActorMaterializer (akka.stream.ActorMaterializer)3 Flow (akka.stream.javadsl.Flow)3 TestHelpers (com.netflix.mantis.master.scheduler.TestHelpers)3 JobClustersManagerActor (io.mantisrx.master.JobClustersManagerActor)3 JobDiscoveryRouteHandler (io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler)3 JobDiscoveryRouteHandlerAkkaImpl (io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl)3 AuditEventSubscriberLoggingImpl (io.mantisrx.master.events.AuditEventSubscriberLoggingImpl)3 LifecycleEventPublisher (io.mantisrx.master.events.LifecycleEventPublisher)3 LifecycleEventPublisherImpl (io.mantisrx.master.events.LifecycleEventPublisherImpl)3 StatusEventSubscriberLoggingImpl (io.mantisrx.master.events.StatusEventSubscriberLoggingImpl)3