Search in sources :

Example 1 with IJobClusterMetadata

use of io.mantisrx.master.jobcluster.IJobClusterMetadata in project mantis by Netflix.

the class SimpleCachedFileStorageProvider method loadJobCluster.

// @Override
public Optional<IJobClusterMetadata> loadJobCluster(String clusterName) {
    File jobClusterFile = new File(JOB_CLUSTERS_DIR + "/" + clusterName);
    if (jobClusterFile.exists()) {
        try (FileInputStream fis = new FileInputStream(jobClusterFile)) {
            IJobClusterMetadata jobClustermeta = mapper.readValue(fis, JobClusterMetadataImpl.class);
            return Optional.ofNullable(jobClustermeta);
        } catch (Exception e) {
            logger.error("skipped file {} due to exception when loading job cluster", jobClusterFile.getName(), e);
        }
    }
    logger.warn("No such job cluster {} ", clusterName);
    return Optional.empty();
}
Also used : File(java.io.File) IJobClusterMetadata(io.mantisrx.master.jobcluster.IJobClusterMetadata) FileInputStream(java.io.FileInputStream) JobClusterAlreadyExistsException(io.mantisrx.server.master.persistence.exceptions.JobClusterAlreadyExistsException) JobAlreadyExistsException(io.mantisrx.server.master.store.JobAlreadyExistsException) IOException(java.io.IOException) InvalidNamedJobException(io.mantisrx.server.master.store.InvalidNamedJobException) InvalidJobException(io.mantisrx.server.master.persistence.exceptions.InvalidJobException)

Example 2 with IJobClusterMetadata

use of io.mantisrx.master.jobcluster.IJobClusterMetadata in project mantis by Netflix.

the class JobClustersManagerActor method initialize.

private void initialize(JobClustersManagerInitialize initMsg) {
    ActorRef sender = getSender();
    try {
        logger.info("In JobClustersManagerActor:initialize");
        this.jobListHelperActor = getContext().actorOf(JobListHelperActor.props(), "JobListHelperActor");
        getContext().watch(jobListHelperActor);
        mantisScheduler = initMsg.getScheduler();
        Map<String, IJobClusterMetadata> jobClusterMap = new HashMap<>();
        this.jobClusterInfoManager = new JobClusterInfoManager(jobStore, mantisScheduler, eventPublisher);
        if (!initMsg.isLoadJobsFromStore()) {
            getContext().become(initializedBehavior);
            sender.tell(new JobClustersManagerInitializeResponse(initMsg.requestId, SUCCESS, "JobClustersManager successfully inited"), getSelf());
        } else {
            List<IJobClusterMetadata> jobClusters = jobStore.loadAllJobClusters();
            logger.info("Read {} job clusters from storage", jobClusters.size());
            List<IMantisJobMetadata> activeJobs = jobStore.loadAllActiveJobs();
            logger.info("Read {} jobs from storage", activeJobs.size());
            List<CompletedJob> completedJobs = jobStore.loadAllCompletedJobs();
            logger.info("Read {} completed jobs from storage", completedJobs.size());
            for (IJobClusterMetadata jobClusterMeta : jobClusters) {
                String clusterName = jobClusterMeta.getJobClusterDefinition().getName();
                jobClusterMap.put(clusterName, jobClusterMeta);
            }
            Map<String, List<IMantisJobMetadata>> clusterToJobMap = new HashMap<>();
            Map<String, List<CompletedJob>> clusterToCompletedJobMap = new HashMap<>();
            // group jobs by cluster
            for (IMantisJobMetadata jobMeta : activeJobs) {
                String clusterName = jobMeta.getClusterName();
                clusterToJobMap.computeIfAbsent(clusterName, k -> new ArrayList<>()).add(jobMeta);
            }
            for (CompletedJob jobMeta : completedJobs) {
                String clusterName = jobMeta.getName();
                clusterToCompletedJobMap.computeIfAbsent(clusterName, k -> new ArrayList<>()).add(jobMeta);
            }
            long masterInitTimeoutSecs = ConfigurationProvider.getConfig().getMasterInitTimeoutSecs();
            long timeout = ((masterInitTimeoutSecs - 60)) > 0 ? (masterInitTimeoutSecs - 60) : masterInitTimeoutSecs;
            Observable.from(jobClusterMap.values()).filter((jobClusterMeta) -> jobClusterMeta != null && jobClusterMeta.getJobClusterDefinition() != null).flatMap((jobClusterMeta) -> {
                Duration t = Duration.ofSeconds(timeout);
                Optional<JobClusterInfo> jobClusterInfoO = jobClusterInfoManager.createClusterActorAndRegister(jobClusterMeta.getJobClusterDefinition());
                if (!jobClusterInfoO.isPresent()) {
                    logger.info("skipping job cluster {} on bootstrap as actor creating failed", jobClusterMeta.getJobClusterDefinition().getName());
                    return Observable.empty();
                }
                JobClusterInfo jobClusterInfo = jobClusterInfoO.get();
                List<IMantisJobMetadata> jobList = Lists.newArrayList();
                List<IMantisJobMetadata> jList = clusterToJobMap.get(jobClusterMeta.getJobClusterDefinition().getName());
                if (jList != null) {
                    jobList.addAll(jList);
                }
                List<CompletedJob> completedJobsList = Lists.newArrayList();
                List<CompletedJob> cList = clusterToCompletedJobMap.get(jobClusterMeta.getJobClusterDefinition().getName());
                if (cList != null) {
                    completedJobsList.addAll(cList);
                }
                JobClusterProto.InitializeJobClusterRequest req = new JobClusterProto.InitializeJobClusterRequest((JobClusterDefinitionImpl) jobClusterMeta.getJobClusterDefinition(), jobClusterMeta.isDisabled(), jobClusterMeta.getLastJobCount(), jobList, completedJobsList, "system", getSelf(), false);
                return jobClusterInfoManager.initializeCluster(jobClusterInfo, req, t);
            }).filter(Objects::nonNull).toBlocking().subscribe((clusterInit) -> {
                logger.info("JobCluster {} inited with code {}", clusterInit.jobClusterName, clusterInit.responseCode);
                numJobClusterInitSuccesses.increment();
            }, (error) -> {
                logger.warn("Exception initializing clusters {}", error.getMessage(), error);
                logger.error("JobClusterManagerActor had errors during initialization NOT transitioning to initialized behavior");
                // getContext().become(initializedBehavior);
                sender.tell(new JobClustersManagerInitializeResponse(initMsg.requestId, SERVER_ERROR, "JobClustersManager  inited with errors"), getSelf());
            }, () -> {
                logger.info("JobClusterManagerActor transitioning to initialized behavior");
                getContext().become(initializedBehavior);
                sender.tell(new JobClustersManagerInitializeResponse(initMsg.requestId, SUCCESS, "JobClustersManager successfully inited"), getSelf());
            });
            getTimers().startPeriodicTimer(CHECK_CLUSTERS_TIMER_KEY, new ReconcileJobCluster(), Duration.ofSeconds(checkAgainInSecs));
            // kick off loading of archived jobs
            logger.info("Kicking off archived job load asynchronously");
            jobStore.loadAllArchivedJobsAsync();
        }
    } catch (Exception e) {
        logger.error("caught exception", e);
        sender.tell(new JobClustersManagerInitializeResponse(initMsg.requestId, SERVER_ERROR, e.getMessage()), getSelf());
    }
    logger.info("JobClustersManagerActor:initialize ends");
}
Also used : JobId(io.mantisrx.server.master.domain.JobId) Terminated(akka.actor.Terminated) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) GetLastSubmittedJobIdStreamResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetLastSubmittedJobIdStreamResponse) ListArchivedWorkersRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListArchivedWorkersRequest) CreateJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.CreateJobClusterRequest) UpdateJobClusterLabelsResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterLabelsResponse) ActorRef(akka.actor.ActorRef) UpdateJobClusterArtifactResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactResponse) DeleteJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DeleteJobClusterRequest) Duration(java.time.Duration) Map(java.util.Map) Schedulers(rx.schedulers.Schedulers) Metrics(io.mantisrx.common.metrics.Metrics) DisableJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterResponse) ListCompletedJobsInClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListCompletedJobsInClusterResponse) Set(java.util.Set) ListCompletedJobsInClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListCompletedJobsInClusterRequest) ScaleStageRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ScaleStageRequest) BaseResponse(io.mantisrx.master.jobcluster.proto.BaseResponse) SUCCESS_CREATED(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.SUCCESS_CREATED) SubmitJobRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.SubmitJobRequest) CompletionStage(java.util.concurrent.CompletionStage) UpdateJobClusterWorkerMigrationStrategyRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterWorkerMigrationStrategyRequest) ListJobIdsRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobIdsRequest) JobHelper(io.mantisrx.master.jobcluster.job.JobHelper) CLIENT_ERROR_CONFLICT(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.CLIENT_ERROR_CONFLICT) ListWorkersRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListWorkersRequest) IJobClusterMetadata(io.mantisrx.master.jobcluster.IJobClusterMetadata) Optional.empty(java.util.Optional.empty) SERVER_ERROR(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.SERVER_ERROR) ListJobClustersRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobClustersRequest) MetricsRegistry(io.mantisrx.common.metrics.MetricsRegistry) ArrayList(java.util.ArrayList) ListJobIdsResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobIdsResponse) SupervisorStrategy(akka.actor.SupervisorStrategy) JobClusterDefinitionImpl(io.mantisrx.server.master.domain.JobClusterDefinitionImpl) WorkerEvent(io.mantisrx.server.master.scheduler.WorkerEvent) GetLastSubmittedJobIdStreamRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetLastSubmittedJobIdStreamRequest) KillJobRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.KillJobRequest) EnableJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.EnableJobClusterRequest) CreateJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.CreateJobClusterResponse) GetJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterRequest) UpdateJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterRequest) CLIENT_ERROR_NOT_FOUND(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.CLIENT_ERROR_NOT_FOUND) UpdateJobClusterWorkerMigrationStrategyResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterWorkerMigrationStrategyResponse) CompletedJob(io.mantisrx.server.master.domain.JobClusterDefinitionImpl.CompletedJob) IMantisJobMetadata(io.mantisrx.master.jobcluster.job.IMantisJobMetadata) KillJobResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.KillJobResponse) ListJobsResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobsResponse) ListJobClustersResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobClustersResponse) EnableJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.EnableJobClusterResponse) IJobClusterDefinition(io.mantisrx.server.master.domain.IJobClusterDefinition) GetJobSchedInfoRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoRequest) JobState(io.mantisrx.master.jobcluster.job.JobState) LoggerFactory(org.slf4j.LoggerFactory) GaugeCallback(io.mantisrx.common.metrics.spectator.GaugeCallback) JobCompletedReason(io.mantisrx.server.core.JobCompletedReason) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) ListJobsRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobsRequest) CLIENT_ERROR(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.CLIENT_ERROR) GetJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobClusterResponse) JobClusterProto(io.mantisrx.master.jobcluster.proto.JobClusterProto) GetLatestJobDiscoveryInfoResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetLatestJobDiscoveryInfoResponse) JobClustersManagerInitialize(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.JobClustersManagerInitialize) ReconcileJobCluster(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ReconcileJobCluster) GetJobDetailsResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobDetailsResponse) JobClusterActor(io.mantisrx.master.jobcluster.JobClusterActor) ResubmitWorkerResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ResubmitWorkerResponse) DisableJobClusterRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest) ResubmitWorkerRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ResubmitWorkerRequest) UpdateJobClusterLabelsRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterLabelsRequest) Collectors(java.util.stream.Collectors) GetJobSchedInfoResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobSchedInfoResponse) Objects(java.util.Objects) List(java.util.List) ActorPaths(akka.actor.ActorPaths) PatternsCS.ask(akka.pattern.PatternsCS.ask) SubmitJobResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.SubmitJobResponse) GetJobDetailsRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetJobDetailsRequest) ConfigurationProvider(io.mantisrx.server.master.config.ConfigurationProvider) Optional(java.util.Optional) Props(akka.actor.Props) UpdateJobClusterSLARequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterSLARequest) ScaleStageResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ScaleStageResponse) UpdateJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterResponse) ListArchivedWorkersResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListArchivedWorkersResponse) HashMap(java.util.HashMap) UpdateJobClusterSLAResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterSLAResponse) MantisActorSupervisorStrategy(io.mantisrx.master.akka.MantisActorSupervisorStrategy) Observable(rx.Observable) DeleteJobClusterResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DeleteJobClusterResponse) ListWorkersResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListWorkersResponse) AbstractActorWithTimers(akka.actor.AbstractActorWithTimers) Counter(io.mantisrx.common.metrics.Counter) Logger(org.slf4j.Logger) Optional.ofNullable(java.util.Optional.ofNullable) GetLatestJobDiscoveryInfoRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.GetLatestJobDiscoveryInfoRequest) SUCCESS(io.mantisrx.master.jobcluster.proto.BaseResponse.ResponseCode.SUCCESS) Lists(io.mantisrx.shaded.com.google.common.collect.Lists) UpdateJobClusterArtifactRequest(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.UpdateJobClusterArtifactRequest) Collections(java.util.Collections) JobClustersManagerInitializeResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.JobClustersManagerInitializeResponse) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) MetricGroupId(io.mantisrx.common.metrics.spectator.MetricGroupId) JobClusterProto(io.mantisrx.master.jobcluster.proto.JobClusterProto) HashMap(java.util.HashMap) ActorRef(akka.actor.ActorRef) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List) ReconcileJobCluster(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ReconcileJobCluster) JobClustersManagerInitializeResponse(io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.JobClustersManagerInitializeResponse) Optional(java.util.Optional) JobClusterDefinitionImpl(io.mantisrx.server.master.domain.JobClusterDefinitionImpl) IMantisJobMetadata(io.mantisrx.master.jobcluster.job.IMantisJobMetadata) Duration(java.time.Duration) CompletedJob(io.mantisrx.server.master.domain.JobClusterDefinitionImpl.CompletedJob) Objects(java.util.Objects) IJobClusterMetadata(io.mantisrx.master.jobcluster.IJobClusterMetadata)

Example 3 with IJobClusterMetadata

use of io.mantisrx.master.jobcluster.IJobClusterMetadata in project mantis by Netflix.

the class SimpleCachedFileStorageProviderTest method testGetAllJobClusters.

@Test
public void testGetAllJobClusters() throws IOException, JobClusterAlreadyExistsException {
    SimpleCachedFileStorageProvider sProvider = new SimpleCachedFileStorageProvider();
    String clusterPrefix = "testGetAllJobClustersCluster";
    for (int i = 0; i < 5; i++) {
        JobClusterDefinitionImpl jobClusterDefn = createFakeJobClusterDefn(clusterPrefix + "_" + i, Lists.newArrayList());
        IJobClusterMetadata jobCluster = new JobClusterMetadataImpl.Builder().withLastJobCount(0).withJobClusterDefinition(jobClusterDefn).build();
        sProvider.createJobCluster(jobCluster);
    }
    List<IJobClusterMetadata> jobClusterList = sProvider.loadAllJobClusters();
    assertTrue(jobClusterList.size() >= 5);
    Map<String, IJobClusterMetadata> clustersMap = Maps.newHashMap();
    for (IJobClusterMetadata cluster : jobClusterList) {
        clustersMap.put(cluster.getJobClusterDefinition().getName(), cluster);
    }
    for (int i = 0; i < 5; i++) {
        assertTrue(clustersMap.containsKey(clusterPrefix + "_" + i));
    }
}
Also used : JobClusterMetadataImpl(io.mantisrx.master.jobcluster.JobClusterMetadataImpl) JobClusterDefinitionImpl(io.mantisrx.server.master.domain.JobClusterDefinitionImpl) IJobClusterMetadata(io.mantisrx.master.jobcluster.IJobClusterMetadata) Test(org.junit.Test)

Example 4 with IJobClusterMetadata

use of io.mantisrx.master.jobcluster.IJobClusterMetadata in project mantis by Netflix.

the class DataFormatAdapterTest method jobClusterMetadataConversionTest.

@Test
public void jobClusterMetadataConversionTest() {
    String artifactName = "artifact1";
    String version = "0.0.1";
    List<Parameter> parameterList = new ArrayList<>();
    Parameter parameter = new Parameter("param1", "value1");
    parameterList.add(parameter);
    List<Label> labels = new ArrayList<>();
    Label label = new Label("label1", "labelvalue1");
    labels.add(label);
    long uAt = 1234l;
    JobClusterConfig jobClusterConfig = new JobClusterConfig.Builder().withArtifactName(artifactName).withSchedulingInfo(DEFAULT_SCHED_INFO).withVersion(version).withUploadedAt(uAt).build();
    String clusterName = "clusterName1";
    JobOwner owner = new JobOwner("Neeraj", "Mantis", "desc", "nma@netflix.com", "repo");
    boolean isReadyForMaster = true;
    SLA sla = new SLA(1, 10, null, null);
    JobClusterDefinitionImpl clusterDefn = new JobClusterDefinitionImpl.Builder().withJobClusterConfig(jobClusterConfig).withName(clusterName).withUser("user1").withIsReadyForJobMaster(isReadyForMaster).withOwner(owner).withMigrationConfig(WorkerMigrationConfig.DEFAULT).withSla(sla).withParameters(parameterList).withLabels(labels).build();
    int lastJobCnt = 10;
    boolean disabled = false;
    IJobClusterMetadata clusterMeta = new JobClusterMetadataImpl.Builder().withJobClusterDefinition(clusterDefn).withLastJobCount(lastJobCnt).withIsDisabled(disabled).build();
    NamedJob namedJob = DataFormatAdapter.convertJobClusterMetadataToNamedJob(clusterMeta);
    assertEquals(disabled, namedJob.getDisabled());
    assertEquals(clusterName, namedJob.getName());
    assertEquals(lastJobCnt, namedJob.getLastJobCount());
    assertEquals(1, namedJob.getLabels().size());
    assertEquals(label, namedJob.getLabels().get(0));
    assertEquals(owner, namedJob.getOwner());
    assertEquals(isReadyForMaster, namedJob.getIsReadyForJobMaster());
    assertEquals(WorkerMigrationConfig.DEFAULT, namedJob.getMigrationConfig());
    // assert parameters
    assertEquals(parameterList.size(), namedJob.getParameters().size());
    assertEquals(parameter, namedJob.getParameters().get(0));
    // assert sla
    assertEquals(sla.getMin(), namedJob.getSla().getMin());
    assertEquals(sla.getMax(), namedJob.getSla().getMax());
    // assert jar info
    assertEquals(1, namedJob.getJars().size());
    // jar info
    NamedJob.Jar jar = namedJob.getJars().get(0);
    assertEquals(uAt, jar.getUploadedAt());
    assertEquals(DEFAULT_SCHED_INFO, jar.getSchedulingInfo());
    assertEquals(version, jar.getVersion());
    assertEquals(artifactName, DataFormatAdapter.extractArtifactName(jar.getUrl()).orElse(""));
    IJobClusterMetadata reconvertedJobCluster = DataFormatAdapter.convertNamedJobToJobClusterMetadata(namedJob);
    assertEquals(disabled, reconvertedJobCluster.isDisabled());
    assertEquals(clusterName, reconvertedJobCluster.getJobClusterDefinition().getName());
    assertEquals(lastJobCnt, reconvertedJobCluster.getLastJobCount());
    assertEquals(1, reconvertedJobCluster.getJobClusterDefinition().getLabels().size());
    assertEquals(label, reconvertedJobCluster.getJobClusterDefinition().getLabels().get(0));
    assertEquals(owner, reconvertedJobCluster.getJobClusterDefinition().getOwner());
    assertEquals(isReadyForMaster, reconvertedJobCluster.getJobClusterDefinition().getIsReadyForJobMaster());
    assertEquals(WorkerMigrationConfig.DEFAULT, reconvertedJobCluster.getJobClusterDefinition().getWorkerMigrationConfig());
    assertEquals(parameterList.size(), reconvertedJobCluster.getJobClusterDefinition().getParameters().size());
    assertEquals(parameter, reconvertedJobCluster.getJobClusterDefinition().getParameters().get(0));
    assertEquals(sla.getMin(), reconvertedJobCluster.getJobClusterDefinition().getSLA().getMin());
    assertEquals(sla.getMax(), reconvertedJobCluster.getJobClusterDefinition().getSLA().getMax());
    JobClusterConfig clusterConfig1 = reconvertedJobCluster.getJobClusterDefinition().getJobClusterConfig();
    assertEquals(uAt, clusterConfig1.getUploadedAt());
    assertEquals(DEFAULT_SCHED_INFO, clusterConfig1.getSchedulingInfo());
    assertEquals(version, clusterConfig1.getVersion());
    assertEquals(artifactName, clusterConfig1.getArtifactName());
}
Also used : JobOwner(io.mantisrx.runtime.JobOwner) ArrayList(java.util.ArrayList) Label(io.mantisrx.common.Label) NamedJob(io.mantisrx.server.master.store.NamedJob) Parameter(io.mantisrx.runtime.parameter.Parameter) IJobClusterMetadata(io.mantisrx.master.jobcluster.IJobClusterMetadata) Test(org.junit.Test)

Example 5 with IJobClusterMetadata

use of io.mantisrx.master.jobcluster.IJobClusterMetadata in project mantis by Netflix.

the class SimpleCachedFileStorageProvider method loadAllJobClusters.

@Override
public List<IJobClusterMetadata> loadAllJobClusters() {
    createDir(JOB_CLUSTERS_DIR);
    File jobClustersDir = new File(JOB_CLUSTERS_DIR);
    final List<IJobClusterMetadata> jobClusterMetadataList = new ArrayList<>();
    for (File jobClusterFile : jobClustersDir.listFiles()) {
        try (FileInputStream fis = new FileInputStream(jobClusterFile)) {
            jobClusterMetadataList.add(mapper.readValue(fis, JobClusterMetadataImpl.class));
        } catch (Exception e) {
            logger.error("skipped file {} due to exception when loading job cluster", jobClusterFile.getName(), e);
        }
    }
    return jobClusterMetadataList;
}
Also used : JobClusterMetadataImpl(io.mantisrx.master.jobcluster.JobClusterMetadataImpl) ArrayList(java.util.ArrayList) File(java.io.File) IJobClusterMetadata(io.mantisrx.master.jobcluster.IJobClusterMetadata) FileInputStream(java.io.FileInputStream) JobClusterAlreadyExistsException(io.mantisrx.server.master.persistence.exceptions.JobClusterAlreadyExistsException) JobAlreadyExistsException(io.mantisrx.server.master.store.JobAlreadyExistsException) IOException(java.io.IOException) InvalidNamedJobException(io.mantisrx.server.master.store.InvalidNamedJobException) InvalidJobException(io.mantisrx.server.master.persistence.exceptions.InvalidJobException)

Aggregations

IJobClusterMetadata (io.mantisrx.master.jobcluster.IJobClusterMetadata)8 JobClusterMetadataImpl (io.mantisrx.master.jobcluster.JobClusterMetadataImpl)4 JobClusterDefinitionImpl (io.mantisrx.server.master.domain.JobClusterDefinitionImpl)4 IOException (java.io.IOException)4 ArrayList (java.util.ArrayList)4 Test (org.junit.Test)4 JobClusterAlreadyExistsException (io.mantisrx.server.master.persistence.exceptions.JobClusterAlreadyExistsException)3 Label (io.mantisrx.common.Label)2 LifecycleEventPublisher (io.mantisrx.master.events.LifecycleEventPublisher)2 IMantisJobMetadata (io.mantisrx.master.jobcluster.job.IMantisJobMetadata)2 InvalidNamedJobException (io.mantisrx.server.master.store.InvalidNamedJobException)2 JobAlreadyExistsException (io.mantisrx.server.master.store.JobAlreadyExistsException)2 AbstractActorWithTimers (akka.actor.AbstractActorWithTimers)1 ActorPaths (akka.actor.ActorPaths)1 ActorRef (akka.actor.ActorRef)1 Props (akka.actor.Props)1 SupervisorStrategy (akka.actor.SupervisorStrategy)1 Terminated (akka.actor.Terminated)1 PatternsCS.ask (akka.pattern.PatternsCS.ask)1 Counter (io.mantisrx.common.metrics.Counter)1