Search in sources :

Example 1 with UpdateJobClusterArtifactRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest in project mantis by Netflix.

the class JobClusterTest method testJobClusterArtifactUpdateNotUniqueFails.

@Test
public void testJobClusterArtifactUpdateNotUniqueFails() throws Exception {
    TestKit probe = new TestKit(system);
    String clusterName = "testJobClusterArtifactUpdateNotUniqueFails";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    UpdateJobClusterArtifactRequest req = new UpdateJobClusterArtifactRequest(clusterName, "a1", "0.0.1", true, "user");
    jobClusterActor.tell(req, probe.getRef());
    UpdateJobClusterArtifactResponse resp = probe.expectMsgClass(UpdateJobClusterArtifactResponse.class);
    assertEquals(CLIENT_ERROR, resp.responseCode);
    assertEquals(jobClusterActor, probe.getLastSender());
    jobClusterActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
    GetJobClusterResponse resp3 = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, resp3.responseCode);
    assertTrue(resp3.getJobCluster() != null);
    System.out.println("Job cluster " + resp3.getJobCluster());
    assertEquals(clusterName, resp3.getJobCluster().get().getName());
    System.out.println("job cluster " + resp3.getJobCluster());
    assertEquals(1, resp3.getJobCluster().get().getJars().size());
    // assertEquals("a1", resp3.getJobCluster().getJobClusterDefinition().getJobClusterConfig().getArtifactName());
    assertEquals("0.0.1", resp3.getJobCluster().get().getLatestVersion());
    verify(jobStoreMock, times(0)).updateJobCluster(any());
    verify(jobStoreMock, times(1)).createJobCluster(any());
}
Also used : JobClusterProto(io.mantisrx.master.jobcluster.proto.JobClusterProto) GetJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterResponse) ActorRef(akka.actor.ActorRef) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) UpdateJobClusterArtifactResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactResponse) TestKit(akka.testkit.javadsl.TestKit) Matchers.anyString(org.mockito.Matchers.anyString) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) UpdateJobClusterArtifactRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest) GetJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterRequest) Test(org.junit.Test)

Example 2 with UpdateJobClusterArtifactRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest in project mantis by Netflix.

the class JobClusterRoute method getJobClusterRoutes.

private Route getJobClusterRoutes() {
    return route(path(segment("api").slash("submit"), () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/submit called {}", request);
        try {
            MantisJobDefinition mjd = Jackson.fromJSON(request, MantisJobDefinition.class);
            logger.debug("job submit request {}", mjd);
            mjd.validate(true);
            Pair<Boolean, String> validationResult = validateSubmitJobRequest(mjd);
            if (!validationResult.first()) {
                jobClusterSubmitError.increment();
                return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"" + validationResult.second() + "\"}");
            }
            jobClusterSubmit.increment();
            return completeWithFuture(jobClusterRouteHandler.submit(JobClusterProtoAdapter.toSubmitJobClusterRequest(mjd)).thenApply(this::toHttpResponse));
        } catch (Exception e) {
            logger.warn("exception in submit job request {}", request, e);
            jobClusterSubmitError.increment();
            return complete(StatusCodes.INTERNAL_SERVER_ERROR, "{\"error\": \"" + e.getMessage() + "\"}");
        }
    }))), pathPrefix(API_V0_JOBCLUSTER, () -> route(post(() -> route(path("create", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), jobClusterDefn -> {
        logger.debug("/api/namedjob/create called {}", jobClusterDefn);
        try {
            final NamedJobDefinition namedJobDefinition = Jackson.fromJSON(jobClusterDefn, NamedJobDefinition.class);
            if (namedJobDefinition == null || namedJobDefinition.getJobDefinition() == null || namedJobDefinition.getJobDefinition().getJobJarFileLocation() == null || namedJobDefinition.getJobDefinition().getName() == null || namedJobDefinition.getJobDefinition().getName().isEmpty()) {
                logger.warn("JobCluster create request must include name and URL {}", jobClusterDefn);
                return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"Job definition must include name and URL\"}");
            }
            final CompletionStage<CreateJobClusterResponse> response = jobClusterRouteHandler.create(JobClusterProtoAdapter.toCreateJobClusterRequest(namedJobDefinition));
            jobClusterCreate.increment();
            return completeWithFuture(response.thenApply(r -> {
                if ((r.responseCode == CLIENT_ERROR || r.responseCode == CLIENT_ERROR_CONFLICT) && r.message.contains("already exists")) {
                    return new CreateJobClusterResponse(r.requestId, SERVER_ERROR, r.message, r.getJobClusterName());
                }
                return r;
            }).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error creating JobCluster {}", jobClusterDefn, e);
            jobClusterCreateError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't read valid json in request: " + e.getMessage());
        } catch (Exception e) {
            logger.warn("Error creating JobCluster {}", jobClusterDefn, e);
            jobClusterCreateError.increment();
            return complete(StatusCodes.INTERNAL_SERVER_ERROR, "{\"error\": " + e.getMessage() + "}");
        }
    }))), path("update", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), jobClusterDefn -> {
        logger.debug("/api/namedjob/update called {}", jobClusterDefn);
        try {
            final NamedJobDefinition namedJobDefinition = Jackson.fromJSON(jobClusterDefn, NamedJobDefinition.class);
            if (namedJobDefinition == null || namedJobDefinition.getJobDefinition() == null || namedJobDefinition.getJobDefinition().getJobJarFileLocation() == null || namedJobDefinition.getJobDefinition().getName() == null || namedJobDefinition.getJobDefinition().getName().isEmpty()) {
                logger.warn("JobCluster update request must include name and URL {}", jobClusterDefn);
                jobClusterCreateUpdateError.increment();
                return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"Job definition must include name and URL\"}");
            }
            final CompletionStage<UpdateJobClusterResponse> response = jobClusterRouteHandler.update(JobClusterProtoAdapter.toUpdateJobClusterRequest(namedJobDefinition));
            jobClusterCreateUpdate.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error updating JobCluster {}", jobClusterDefn, e);
            jobClusterCreateUpdateError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't read valid json in request: " + e.getMessage());
        } catch (Exception e) {
            logger.warn("Error updating JobCluster {}", jobClusterDefn, e);
            jobClusterCreateUpdateError.increment();
            return complete(StatusCodes.INTERNAL_SERVER_ERROR, "{\"error\": " + e.getMessage() + "}");
        }
    }))), path("delete", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), deleteReq -> {
        logger.debug("/api/namedjob/delete called {}", deleteReq);
        try {
            final DeleteJobClusterRequest deleteJobClusterRequest = Jackson.fromJSON(deleteReq, DeleteJobClusterRequest.class);
            final CompletionStage<DeleteJobClusterResponse> response = jobClusterRouteHandler.delete(deleteJobClusterRequest);
            jobClusterDelete.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error deleting JobCluster {}", deleteReq, e);
            jobClusterDeleteError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("disable", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/disable called {}", request);
        try {
            final DisableJobClusterRequest disableJobClusterRequest = Jackson.fromJSON(request, DisableJobClusterRequest.class);
            final CompletionStage<DisableJobClusterResponse> response = jobClusterRouteHandler.disable(disableJobClusterRequest);
            jobClusterDisable.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error disabling JobCluster {}", request, e);
            jobClusterDisableError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("enable", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/enable called {}", request);
        try {
            final EnableJobClusterRequest enableJobClusterRequest = Jackson.fromJSON(request, EnableJobClusterRequest.class);
            final CompletionStage<EnableJobClusterResponse> response = jobClusterRouteHandler.enable(enableJobClusterRequest);
            jobClusterEnable.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error enabling JobCluster {}", request, e);
            jobClusterEnableError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("quickupdate", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/quickupdate called {}", request);
        try {
            final UpdateJobClusterArtifactRequest updateJobClusterArtifactRequest = Jackson.fromJSON(request, UpdateJobClusterArtifactRequest.class);
            final CompletionStage<UpdateJobClusterArtifactResponse> response = jobClusterRouteHandler.updateArtifact(updateJobClusterArtifactRequest);
            jobClusterQuickupdate.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error on quickupdate for JobCluster {}", request, e);
            jobClusterQuickupdateError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("updatelabels", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/updatelabels called {}", request);
        try {
            final UpdateJobClusterLabelsRequest updateJobClusterLabelsRequest = Jackson.fromJSON(request, UpdateJobClusterLabelsRequest.class);
            jobClusterUpdateLabel.increment();
            return completeWithFuture(jobClusterRouteHandler.updateLabels(updateJobClusterLabelsRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error updating labels for JobCluster {}", request, e);
            jobClusterUpdateLabelError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("updatesla", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/updatesla called {}", request);
        jobClusterUpdateSla.increment();
        try {
            final UpdateJobClusterSLARequest updateJobClusterSLARequest = Jackson.fromJSON(request, UpdateJobClusterSLARequest.class);
            return completeWithFuture(jobClusterRouteHandler.updateSLA(updateJobClusterSLARequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error updating SLA for JobCluster {}", request, e);
            jobClusterUpdateSlaError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("migratestrategy", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/migratestrategy called {}", request);
        try {
            final UpdateJobClusterWorkerMigrationStrategyRequest updateMigrateStrategyReq = Jackson.fromJSON(request, UpdateJobClusterWorkerMigrationStrategyRequest.class);
            return completeWithFuture(jobClusterRouteHandler.updateWorkerMigrateStrategy(updateMigrateStrategyReq).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error updating migrate strategy for JobCluster {}", request, e);
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("quicksubmit", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/quicksubmit called {}", request);
        try {
            final JobClusterManagerProto.SubmitJobRequest submitJobRequest = Jackson.fromJSON(request, JobClusterManagerProto.SubmitJobRequest.class);
            return completeWithFuture(jobClusterRouteHandler.submit(submitJobRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error on quick submit for JobCluster {}", request, e);
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))))), get(() -> route(pathPrefix("list", () -> route(pathEndOrSingleSlash(() -> {
        logger.debug("/api/namedjob/list called");
        jobClusterListGET.increment();
        return alwaysCache(cache, requestUriKeyer, () -> extractUri(uri -> completeAsync(jobClusterRouteHandler.getAllJobClusters(new ListJobClustersRequest()), resp -> completeOK(resp.getJobClusters().stream().map(jobClusterMetadataView -> JobClusterProtoAdapter.toJobClusterInfo(jobClusterMetadataView)).collect(Collectors.toList()), Jackson.marshaller()), resp -> completeOK(Collections.emptyList(), Jackson.marshaller()))));
    }), path(PathMatchers.segment(), (jobCluster) -> {
        if (logger.isDebugEnabled()) {
            logger.debug("/api/namedjob/list/{} called", jobCluster);
        }
        jobClusterListClusterGET.increment();
        return completeAsync(jobClusterRouteHandler.getJobClusterDetails(new JobClusterManagerProto.GetJobClusterRequest(jobCluster)), resp -> completeOK(resp.getJobCluster().map(jc -> Arrays.asList(jc)).orElse(Collections.emptyList()), Jackson.marshaller()), resp -> completeOK(Collections.emptyList(), Jackson.marshaller()));
    }))), path(segment("listJobIds").slash(PathMatchers.segment()), (jobCluster) -> {
        logger.debug("/api/namedjob/listJobIds/{} called", jobCluster);
        jobClusterListJobIdGET.increment();
        return jobClusterListRoute(jobCluster);
    }), path("listJobIds", () -> {
        logger.debug("/api/namedjob/listJobIds called");
        return complete(StatusCodes.BAD_REQUEST, "Specify the Job cluster name '/api/namedjob/listJobIds/<JobClusterName>' to list the job Ids");
    }))))));
}
Also used : Strings(io.mantisrx.shaded.com.google.common.base.Strings) EnableJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.EnableJobClusterResponse) Uri(akka.http.javadsl.model.Uri) MasterConfiguration(io.mantisrx.server.master.config.MasterConfiguration) Arrays(java.util.Arrays) JavaPartialFunction(akka.japi.JavaPartialFunction) Cache(akka.http.caching.javadsl.Cache) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) LoggerFactory(org.slf4j.LoggerFactory) LfuCache(akka.http.caching.LfuCache) RequestContext(akka.http.javadsl.server.RequestContext) StageSchedulingInfo(io.mantisrx.runtime.descriptor.StageSchedulingInfo) PathMatchers.segment(akka.http.javadsl.server.PathMatchers.segment) MantisJobDefinition(io.mantisrx.runtime.MantisJobDefinition) CLIENT_ERROR(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.CLIENT_ERROR) StringUnmarshallers(akka.http.javadsl.unmarshalling.StringUnmarshallers) PathMatcher0(akka.http.javadsl.server.PathMatcher0) LfuCacheSettings(akka.http.caching.javadsl.LfuCacheSettings) UpdateJobClusterArtifactResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactResponse) DeleteJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DeleteJobClusterRequest) Map(java.util.Map) JobClusterProtoAdapter(io.mantisrx.master.api.akka.route.proto.JobClusterProtoAdapter) Unmarshaller(akka.http.javadsl.unmarshalling.Unmarshaller) Metrics(io.mantisrx.common.metrics.Metrics) DisableJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest) DisableJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterResponse) Jackson(io.mantisrx.master.api.akka.route.Jackson) UpdateJobClusterLabelsRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterLabelsRequest) HttpMethods(akka.http.javadsl.model.HttpMethods) Collectors(java.util.stream.Collectors) ExceptionHandler(akka.http.javadsl.server.ExceptionHandler) JobClusterRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler) Duration(scala.concurrent.duration.Duration) CompletionStage(java.util.concurrent.CompletionStage) UpdateJobClusterWorkerMigrationStrategyRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterWorkerMigrationStrategyRequest) JobRouteUtils.createListJobIdsRequest(io.mantisrx.master.api.akka.route.utils.JobRouteUtils.createListJobIdsRequest) CachingDirectives.alwaysCache(akka.http.javadsl.server.directives.CachingDirectives.alwaysCache) ActorSystem(akka.actor.ActorSystem) ConfigurationProvider(io.mantisrx.server.master.config.ConfigurationProvider) StageScalingPolicy(io.mantisrx.runtime.descriptor.StageScalingPolicy) Optional(java.util.Optional) CLIENT_ERROR_CONFLICT(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.CLIENT_ERROR_CONFLICT) UpdateJobClusterSLARequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterSLARequest) SERVER_ERROR(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.SERVER_ERROR) UpdateJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterResponse) ListJobClustersRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobClustersRequest) Route(akka.http.javadsl.server.Route) MetricsRegistry(io.mantisrx.common.metrics.MetricsRegistry) StatusCodes(akka.http.javadsl.model.StatusCodes) Function(java.util.function.Function) PathMatchers(akka.http.javadsl.server.PathMatchers) DeleteJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DeleteJobClusterResponse) SchedulingInfo(io.mantisrx.runtime.descriptor.SchedulingInfo) Counter(io.mantisrx.common.metrics.Counter) Logger(org.slf4j.Logger) RouteResult(akka.http.javadsl.server.RouteResult) HttpHeader(akka.http.javadsl.model.HttpHeader) HttpRequest(akka.http.javadsl.model.HttpRequest) NamedJobDefinition(io.mantisrx.runtime.NamedJobDefinition) EnableJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.EnableJobClusterRequest) IOException(java.io.IOException) CreateJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.CreateJobClusterResponse) Pair(akka.japi.Pair) TimeUnit(java.util.concurrent.TimeUnit) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto) UpdateJobClusterArtifactRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest) CachingSettings(akka.http.caching.javadsl.CachingSettings) Collections(java.util.Collections) UpdateJobClusterLabelsRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterLabelsRequest) UpdateJobClusterSLARequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterSLARequest) MantisJobDefinition(io.mantisrx.runtime.MantisJobDefinition) ListJobClustersRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobClustersRequest) IOException(java.io.IOException) IOException(java.io.IOException) DisableJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest) UpdateJobClusterArtifactRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest) EnableJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.EnableJobClusterRequest) UpdateJobClusterWorkerMigrationStrategyRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterWorkerMigrationStrategyRequest) DeleteJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DeleteJobClusterRequest) NamedJobDefinition(io.mantisrx.runtime.NamedJobDefinition) CreateJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.CreateJobClusterResponse) CompletionStage(java.util.concurrent.CompletionStage) Pair(akka.japi.Pair) JobClusterManagerProto(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto)

Example 3 with UpdateJobClusterArtifactRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest in project mantis by Netflix.

the class JobClusterTest method testJobClusterArtifactUpdateMultipleTimes.

@Test
public void testJobClusterArtifactUpdateMultipleTimes() throws Exception {
    TestKit probe = new TestKit(system);
    String clusterName = "testJobClusterArtifactUpdateMultipleTimes";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    UpdateJobClusterArtifactRequest req = new UpdateJobClusterArtifactRequest(clusterName, "a1", "1.0.1", true, "user");
    jobClusterActor.tell(req, probe.getRef());
    UpdateJobClusterArtifactResponse resp = probe.expectMsgClass(UpdateJobClusterArtifactResponse.class);
    assertEquals(SUCCESS, resp.responseCode);
    assertEquals(jobClusterActor, probe.getLastSender());
    jobClusterActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
    GetJobClusterResponse resp3 = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, resp3.responseCode);
    assertTrue(resp3.getJobCluster() != null);
    System.out.println("Job cluster " + resp3.getJobCluster());
    assertEquals(clusterName, resp3.getJobCluster().get().getName());
    System.out.println("Updated job cluster " + resp3.getJobCluster());
    // assertEquals("a1", resp3.getJobCluster().getJobClusterDefinition().getJobClusterConfig().getArtifactName());
    assertEquals("1.0.1", resp3.getJobCluster().get().getLatestVersion());
    List<NamedJob.Jar> jars = resp3.getJobCluster().get().getJars();
    System.out.println("jars --> " + jars);
    assertEquals(2, jars.size());
    // Update again
    req = new UpdateJobClusterArtifactRequest(clusterName, "a2", "1.0.3", true, "user");
    jobClusterActor.tell(req, probe.getRef());
    resp = probe.expectMsgClass(UpdateJobClusterArtifactResponse.class);
    assertEquals(SUCCESS, resp.responseCode);
    assertEquals(jobClusterActor, probe.getLastSender());
    jobClusterActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
    resp3 = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, resp3.responseCode);
    assertTrue(resp3.getJobCluster() != null);
    System.out.println("Job cluster " + resp3.getJobCluster());
    assertEquals(clusterName, resp3.getJobCluster().get().getName());
    System.out.println("Updated job cluster " + resp3.getJobCluster());
    // assertEquals("a1", resp3.getJobCluster().getJobClusterDefinition().getJobClusterConfig().getArtifactName());
    assertEquals("1.0.3", resp3.getJobCluster().get().getLatestVersion());
    jars = resp3.getJobCluster().get().getJars();
    System.out.println("jars --> " + jars);
    assertEquals(3, jars.size());
    verify(jobStoreMock, times(2)).updateJobCluster(any());
    verify(jobStoreMock, times(1)).createJobCluster(any());
}
Also used : JobClusterProto(io.mantisrx.master.jobcluster.proto.JobClusterProto) GetJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterResponse) ActorRef(akka.actor.ActorRef) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) UpdateJobClusterArtifactResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactResponse) TestKit(akka.testkit.javadsl.TestKit) Matchers.anyString(org.mockito.Matchers.anyString) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) UpdateJobClusterArtifactRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest) GetJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterRequest) Test(org.junit.Test)

Example 4 with UpdateJobClusterArtifactRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest in project mantis by Netflix.

the class JobClusterTest method testUpdateJobClusterArtifactWithAutoSubmit.

@Test
public void testUpdateJobClusterArtifactWithAutoSubmit() {
    TestKit probe = new TestKit(system);
    try {
        String clusterName = "testUpdateJobClusterArtifactWithAutoSubmit";
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        // when running concurrently with testGetJobDetailsForArchivedJob the following mock return is needed to avoid null pointer exception.
        when(jobStoreMock.getArchivedJob(anyString())).thenReturn(empty());
        SLA sla = new SLA(1, 1, null, null);
        final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, Lists.newArrayList(), sla);
        ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
        jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
        JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
        assertEquals(SUCCESS, createResp.responseCode);
        // submit job with different scheduling info instance count compared to cluster default one.
        final int job1InstanceCnt = 3;
        final JobDefinition jobDefn = createJob(clusterName, MantisJobDurationType.Transient, new SchedulingInfo.Builder().numberOfStages(1).addStage(fakeJobCluster.getJobClusterConfig().getSchedulingInfo().forStage(1).toBuilder().numberOfInstances(job1InstanceCnt).build()).build());
        String jobId = clusterName + "-1";
        jobClusterActor.tell(new SubmitJobRequest(clusterName, "user", Optional.ofNullable(jobDefn)), probe.getRef());
        SubmitJobResponse submitResponse = probe.expectMsgClass(SubmitJobResponse.class);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, 1, new WorkerId(clusterName, jobId, 0, 1));
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, BaseResponse.ResponseCode.SUCCESS, JobState.Accepted);
        // Update artifact with skip submit = false
        String artifact = "newartifact.zip";
        String version = "0.0.2";
        jobClusterActor.tell(new UpdateJobClusterArtifactRequest(clusterName, artifact, version, false, user), probe.getRef());
        UpdateJobClusterArtifactResponse resp = probe.expectMsgClass(UpdateJobClusterArtifactResponse.class);
        // ensure new job was launched
        String jobId2 = clusterName + "-2";
        assertTrue(JobTestHelper.verifyJobStatusWithPolling(probe, jobClusterActor, jobId2, JobState.Accepted));
        // send it worker events to move it to started state
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId2, 1, new WorkerId(clusterName, jobId2, 0, 1));
        jobClusterActor.tell(new GetJobDetailsRequest("nj", JobId.fromId(jobId2).get()), probe.getRef());
        GetJobDetailsResponse detailsResp = probe.expectMsgClass(Duration.ofSeconds(5), GetJobDetailsResponse.class);
        assertEquals(JobState.Accepted, detailsResp.getJobMetadata().get().getState());
        assertEquals(artifact, detailsResp.getJobMetadata().get().getArtifactName());
        // verify newly launched job inherited instance count from previous job instance.
        AtomicBoolean hasStage = new AtomicBoolean(false);
        detailsResp.getJobMetadata().get().getSchedulingInfo().getStages().forEach((stageId, stageInfo) -> {
            hasStage.set(true);
            assertEquals(job1InstanceCnt, detailsResp.getJobMetadata().get().getSchedulingInfo().forStage(stageId).getNumberOfInstances());
        });
        assertTrue(hasStage.get());
        assertTrue(JobTestHelper.verifyJobStatusWithPolling(probe, jobClusterActor, jobId2, JobState.Accepted));
    } catch (InvalidJobException e) {
        e.printStackTrace();
    }
}
Also used : JobClusterProto(io.mantisrx.master.jobcluster.proto.JobClusterProto) ActorRef(akka.actor.ActorRef) SubmitJobRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.SubmitJobRequest) GetJobDetailsRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobDetailsRequest) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) UpdateJobClusterArtifactResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactResponse) TestKit(akka.testkit.javadsl.TestKit) Matchers.anyString(org.mockito.Matchers.anyString) WorkerId(io.mantisrx.server.core.domain.WorkerId) GetJobDetailsResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobDetailsResponse) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) UpdateJobClusterArtifactRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest) SubmitJobResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.SubmitJobResponse) InvalidJobException(io.mantisrx.runtime.command.InvalidJobException) Test(org.junit.Test)

Example 5 with UpdateJobClusterArtifactRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest in project mantis by Netflix.

the class JobClusterTest method testJobClusterArtifactUpdate.

@Test
public void testJobClusterArtifactUpdate() throws Exception {
    TestKit probe = new TestKit(system);
    String clusterName = "testJobClusterArtifactUpdate";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    UpdateJobClusterArtifactRequest req = new UpdateJobClusterArtifactRequest(clusterName, "a1", "1.0.1", true, "user");
    jobClusterActor.tell(req, probe.getRef());
    UpdateJobClusterArtifactResponse resp = probe.expectMsgClass(UpdateJobClusterArtifactResponse.class);
    assertEquals(SUCCESS, resp.responseCode);
    assertEquals(jobClusterActor, probe.getLastSender());
    jobClusterActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
    GetJobClusterResponse resp3 = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, resp3.responseCode);
    assertTrue(resp3.getJobCluster() != null);
    System.out.println("Job cluster " + resp3.getJobCluster());
    assertEquals(clusterName, resp3.getJobCluster().get().getName());
    System.out.println("Updated job cluster " + resp3.getJobCluster());
    assertEquals(2, resp3.getJobCluster().get().getJars().size());
    // assertEquals("a1", resp3.getJobCluster().getJobClusterDefinition().getJobClusterConfig().getArtifactName());
    assertEquals("1.0.1", resp3.getJobCluster().get().getLatestVersion());
    List<NamedJob.Jar> jars = resp3.getJobCluster().get().getJars();
    assertTrue(jars.get(jars.size() - 1).getUploadedAt() != -1);
    verify(jobStoreMock, times(1)).updateJobCluster(any());
    verify(jobStoreMock, times(1)).createJobCluster(any());
}
Also used : JobClusterProto(io.mantisrx.master.jobcluster.proto.JobClusterProto) GetJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterResponse) ActorRef(akka.actor.ActorRef) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) UpdateJobClusterArtifactResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactResponse) TestKit(akka.testkit.javadsl.TestKit) Matchers.anyString(org.mockito.Matchers.anyString) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) UpdateJobClusterArtifactRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest) GetJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterRequest) Test(org.junit.Test)

Aggregations

UpdateJobClusterArtifactRequest (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest)6 TestKit (akka.testkit.javadsl.TestKit)5 UpdateJobClusterArtifactResponse (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactResponse)5 Test (org.junit.Test)5 ActorRef (akka.actor.ActorRef)4 GetJobClusterRequest (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterRequest)4 GetJobClusterResponse (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterResponse)4 JobClusterProto (io.mantisrx.master.jobcluster.proto.JobClusterProto)4 MantisJobStore (io.mantisrx.server.master.persistence.MantisJobStore)4 MantisScheduler (io.mantisrx.server.master.scheduler.MantisScheduler)4 Matchers.anyString (org.mockito.Matchers.anyString)4 JobClusterManagerProto (io.mantisrx.master.jobcluster.proto.JobClusterManagerProto)2 ActorSystem (akka.actor.ActorSystem)1 LfuCache (akka.http.caching.LfuCache)1 Cache (akka.http.caching.javadsl.Cache)1 CachingSettings (akka.http.caching.javadsl.CachingSettings)1 LfuCacheSettings (akka.http.caching.javadsl.LfuCacheSettings)1 HttpHeader (akka.http.javadsl.model.HttpHeader)1 HttpMethods (akka.http.javadsl.model.HttpMethods)1 HttpRequest (akka.http.javadsl.model.HttpRequest)1