Examples with RunInstance com.epam.pipeline.entity.pipeline.RunInstance used on opensource projects

Example 16 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class PipelineRunManager method verifyPipelineRunForPauseResume.

private void verifyPipelineRunForPauseResume(PipelineRun pipelineRun, Long runId) {
    Assert.notNull(pipelineRun, messageHelper.getMessage(MessageConstants.ERROR_RUN_PIPELINES_NOT_FOUND, runId));
    Assert.notNull(pipelineRun.getId(), messageHelper.getMessage(MessageConstants.ERROR_PIPELINE_RUN_ID_NOT_FOUND, runId));
    Assert.notNull(pipelineRun.getInstance(), messageHelper.getMessage(MessageConstants.ERROR_INSTANCE_NOT_FOUND, runId));
    RunInstance instance = pipelineRun.getInstance();
    Assert.notNull(instance.getNodeId(), messageHelper.getMessage(MessageConstants.ERROR_INSTANCE_ID_NOT_FOUND, runId));
    Assert.notNull(instance.getNodeIP(), messageHelper.getMessage(MessageConstants.ERROR_INSTANCE_IP_NOT_FOUND, runId));
    Assert.isTrue(!instance.getSpot(), messageHelper.getMessage(MessageConstants.ERROR_ON_DEMAND_REQUIRED));
    Assert.notNull(pipelineRun.getPodId(), messageHelper.getMessage(MessageConstants.ERROR_POD_ID_NOT_FOUND, runId));
}

Example 17 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class PipelineConfigurationManager method getConfigurationFromRun.

public PipelineConfiguration getConfigurationFromRun(PipelineRun run) {
    PipelineConfiguration configuration = new PipelineConfiguration();
    configuration.setParameters(run.convertParamsToMap());
    configuration.setTimeout(run.getTimeout());
    configuration.setNodeCount(run.getNodeCount());
    configuration.setCmdTemplate(run.getCmdTemplate());
    configuration.setDockerImage(run.getDockerImage());
    RunInstance instance = run.getInstance();
    if (instance != null) {
        configuration.setInstanceDisk(String.valueOf(instance.getEffectiveNodeDisk()));
        configuration.setInstanceType(instance.getNodeType());
        configuration.setIsSpot(instance.getSpot());
        configuration.setInstanceImage(instance.getNodeImage());
        configuration.setAwsRegionId(awsRegionManager.loadByAwsRegionName(instance.getAwsRegionId()).getId());
    }
    setEndpointsErasure(configuration);
    if (run.getPipelineId() != null) {
        try {
            ConfigurationEntry entry = pipelineVersionManager.loadConfigurationEntry(run.getPipelineId(), run.getVersion(), run.getConfigName());
            PipelineConfiguration defaultConfiguration = entry.getConfiguration();
            configuration.setEnvironmentParams(defaultConfiguration.getEnvironmentParams());
            configuration.setMainClass(defaultConfiguration.getMainClass());
            configuration.setMainFile(defaultConfiguration.getMainFile());
        } catch (GitClientException e) {
            log.error(e.getMessage(), e);
        }
        configuration.setGitCredentials(gitManager.getGitCredentials(run.getPipelineId()));
    }
    configuration.buildEnvVariables();
    return configuration;
}

Example 18 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class AutoscaleManagerTest method setUp.

@Before
public void setUp() throws Exception {
    MockitoAnnotations.initMocks(this);
    autoscaleManager = new AutoscaleManager(pipelineRunManager, executorService, clusterManager, nodesManager, kubernetesManager, preferenceManager, TEST_KUBE_NAMESPACE);
    when(executorService.getExecutorService()).thenReturn(new CurrentThreadExecutorService());
    // Mock preferences
    when(preferenceManager.getPreference(SystemPreferences.CLUSTER_NODEUP_RETRY_COUNT)).thenReturn(2);
    when(preferenceManager.getPreference(SystemPreferences.CLUSTER_SPOT_MAX_ATTEMPTS)).thenReturn(1);
    when(preferenceManager.getPreference(SystemPreferences.CLUSTER_SPOT)).thenReturn(true);
    when(preferenceManager.getPreference(SystemPreferences.CLUSTER_MAX_SIZE)).thenReturn(1);
    when(preferenceManager.getPreference(SystemPreferences.CLUSTER_NODEUP_MAX_THREADS)).thenReturn(1);
    when(preferenceManager.getPreference(SystemPreferences.CLUSTER_MIN_SIZE)).thenReturn(SystemPreferences.CLUSTER_MIN_SIZE.getDefaultValue());
    when(preferenceManager.getPreference(SystemPreferences.CLUSTER_RANDOM_SCHEDULING)).thenReturn(SystemPreferences.CLUSTER_RANDOM_SCHEDULING.getDefaultValue());
    when(preferenceManager.getPreference(SystemPreferences.CLUSTER_HIGH_NON_BATCH_PRIORITY)).thenReturn(SystemPreferences.CLUSTER_HIGH_NON_BATCH_PRIORITY.getDefaultValue());
    when(kubernetesManager.getKubernetesClient(any(Config.class))).thenReturn(kubernetesClient);
    // Mock no nodes in cluster
    NonNamespaceOperation<Node, NodeList, DoneableNode, Resource<Node, DoneableNode>> mockNodes = new KubernetesTestUtils.MockNodes().mockWithLabel(KubernetesConstants.RUN_ID_LABEL).mockWithoutLabel(KubernetesConstants.PAUSED_NODE_LABEL).mockNodeList(Collections.emptyList()).and().getMockedEntity();
    when(kubernetesClient.nodes()).thenReturn(mockNodes);
    // Mock one unsceduled pod
    Pod unscheduledPipelinePod = new Pod();
    ObjectMeta metadata = new ObjectMeta();
    metadata.setLabels(Collections.singletonMap(KubernetesConstants.RUN_ID_LABEL, TEST_RUN_ID.toString()));
    unscheduledPipelinePod.setMetadata(metadata);
    PodStatus status = new PodStatus();
    status.setPhase("Pending");
    PodCondition condition = new PodCondition();
    condition.setReason(KubernetesConstants.POD_UNSCHEDULABLE);
    status.setConditions(Collections.singletonList(condition));
    unscheduledPipelinePod.setStatus(status);
    PipelineRun testRun = new PipelineRun();
    testRun.setStatus(TaskStatus.RUNNING);
    testRun.setPipelineRunParameters(Collections.emptyList());
    RunInstance spotInstance = new RunInstance();
    spotInstance.setSpot(true);
    testRun.setInstance(spotInstance);
    when(pipelineRunManager.loadPipelineRun(Mockito.eq(TEST_RUN_ID))).thenReturn(testRun);
    when(clusterManager.fillInstance(any(RunInstance.class))).thenAnswer(invocation -> invocation.getArguments()[0]);
    MixedOperation<Pod, PodList, DoneablePod, PodResource<Pod, DoneablePod>> mockPods = new KubernetesTestUtils.MockPods().mockNamespace(TEST_KUBE_NAMESPACE).mockWithLabel("type", "pipeline").mockWithLabel(KubernetesConstants.RUN_ID_LABEL).mockPodList(Collections.singletonList(unscheduledPipelinePod)).and().getMockedEntity();
    when(kubernetesClient.pods()).thenReturn(mockPods);
}

Example 19 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class ResourceMonitoringManagerTest method setUp.

@Before
public void setUp() throws Exception {
    MockitoAnnotations.initMocks(this);
    resourceMonitoringManager = new ResourceMonitoringManager(pipelineRunManager, preferenceManager, notificationManager, instanceOfferManager, monitoringESDao, taskScheduler, messageHelper);
    when(preferenceManager.getObservablePreference(SystemPreferences.SYSTEM_RESOURCE_MONITORING_PERIOD)).thenReturn(Observable.empty());
    when(preferenceManager.getPreference(SystemPreferences.SYSTEM_RESOURCE_MONITORING_PERIOD)).thenReturn(TEST_RESOURCE_MONITORING_DELAY);
    when(preferenceManager.getPreference(SystemPreferences.SYSTEM_IDLE_CPU_THRESHOLD_PERCENT)).thenReturn(TEST_IDLE_THRESHOLD_PERCENT);
    when(preferenceManager.getPreference(SystemPreferences.SYSTEM_IDLE_ACTION_TIMEOUT_MINUTES)).thenReturn(1);
    when(preferenceManager.getPreference(SystemPreferences.SYSTEM_MAX_IDLE_TIMEOUT_MINUTES)).thenReturn(TEST_MAX_IDLE_MONITORING_TIMEOUT);
    testType = new InstanceType();
    testType.setVCPU(2);
    testType.setName("t1.test");
    BehaviorSubject<List<InstanceType>> mockSubject = BehaviorSubject.createDefault(Collections.singletonList(testType));
    when(instanceOfferManager.getAllInstanceTypesObservable()).thenReturn(mockSubject);
    RunInstance spotInstance = new RunInstance(testType.getName(), 0, 0, null, null, null, null, true, null);
    okayRun = new PipelineRun();
    okayRun.setInstance(spotInstance);
    okayRun.setPodId("okay-pod");
    okayRun.setId(TEST_OK_RUN_ID);
    okayRun.setStartDate(new Date(Instant.now().minus(TEST_MAX_IDLE_MONITORING_TIMEOUT + 1, ChronoUnit.MINUTES).toEpochMilli()));
    okayRun.setProlongedAtTime(DateUtils.nowUTC().minus(TEST_MAX_IDLE_MONITORING_TIMEOUT + 1, ChronoUnit.MINUTES));
    idleSpotRun = new PipelineRun();
    idleSpotRun.setInstance(spotInstance);
    idleSpotRun.setPodId("idle-spot");
    idleSpotRun.setId(TEST_IDLE_SPOT_RUN_ID);
    idleSpotRun.setStartDate(new Date(Instant.now().minus(TEST_MAX_IDLE_MONITORING_TIMEOUT + 1, ChronoUnit.MINUTES).toEpochMilli()));
    idleSpotRun.setProlongedAtTime(DateUtils.nowUTC().minus(TEST_MAX_IDLE_MONITORING_TIMEOUT + 1, ChronoUnit.MINUTES));
    idleOnDemandRun = new PipelineRun();
    idleOnDemandRun.setInstance(new RunInstance(testType.getName(), 0, 0, null, null, null, null, false, null));
    idleOnDemandRun.setPodId("idle-on-demand");
    idleOnDemandRun.setId(TEST_IDLE_ON_DEMAND_RUN_ID);
    idleOnDemandRun.setStartDate(new Date(Instant.now().minus(TEST_MAX_IDLE_MONITORING_TIMEOUT + 1, ChronoUnit.MINUTES).toEpochMilli()));
    idleOnDemandRun.setProlongedAtTime(DateUtils.nowUTC().minus(TEST_MAX_IDLE_MONITORING_TIMEOUT + 1, ChronoUnit.MINUTES));
    idleRunToProlong = new PipelineRun();
    idleRunToProlong.setInstance(new RunInstance(testType.getName(), 0, 0, null, null, null, null, false, null));
    idleRunToProlong.setPodId("idle-to-prolong");
    idleRunToProlong.setId(TEST_IDLE_RUN_TO_PROLONG_ID);
    idleRunToProlong.setStartDate(new Date(Instant.now().minus(TEST_MAX_IDLE_MONITORING_TIMEOUT + 1, ChronoUnit.MINUTES).toEpochMilli()));
    idleRunToProlong.setProlongedAtTime(DateUtils.nowUTC().minus(TEST_MAX_IDLE_MONITORING_TIMEOUT + 1, ChronoUnit.MINUTES));
    mockStats = new HashMap<>();
    // in milicores, equals 80% of core load, per 2 cores,
    mockStats.put(okayRun.getPodId(), TEST_OK_RUN_CPU_LOAD);
    // should be = 40% load
    mockStats.put(idleSpotRun.getPodId(), TEST_IDLE_SPOT_RUN_CPU_LOAD);
    mockStats.put(idleOnDemandRun.getPodId(), TEST_IDLE_ON_DEMAND_RUN_CPU_LOAD);
    when(monitoringESDao.loadCpuUsageRateMetrics(any(), any(LocalDateTime.class), any(LocalDateTime.class))).thenReturn(mockStats);
    resourceMonitoringManager.init();
    verify(taskScheduler).scheduleWithFixedDelay(any(), eq(TEST_RESOURCE_MONITORING_DELAY.longValue()));
    Assert.assertNotNull(Whitebox.getInternalState(resourceMonitoringManager, "instanceTypeMap"));
}

Example 20 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class RunMapperTest method shouldMapRun.

@Test
void shouldMapRun() throws IOException {
    PipelineRunMapper mapper = new PipelineRunMapper();
    PipelineRunWithLog pipelineRunWithLog = new PipelineRunWithLog();
    RunInstance instance = new RunInstance();
    instance.setNodeType("type");
    instance.setAwsRegionId(TEST_REGION);
    instance.setSpot(true);
    instance.setNodeDisk(NODE_DISK);
    instance.setNodeId("id");
    instance.setNodeImage(TEST_PATH);
    instance.setNodeName(TEST_NAME);
    RunStatus runStatus = new RunStatus();
    runStatus.setRunId(1L);
    runStatus.setStatus(TaskStatus.SUCCESS);
    PipelineRunParameter parameter = new PipelineRunParameter();
    parameter.setName(TEST_NAME);
    parameter.setValue(TEST_VALUE);
    PipelineRun run = new PipelineRun();
    run.setId(1L);
    run.setName(TEST_NAME);
    run.setPipelineName(TEST_NAME);
    run.setInstance(instance);
    run.setStatus(TaskStatus.SUCCESS);
    run.setPipelineName(TEST_NAME);
    run.setVersion(TEST_VERSION);
    run.setRunStatuses(Collections.singletonList(runStatus));
    run.setPricePerHour(PRICE);
    run.setPipelineRunParameters(Collections.singletonList(parameter));
    RunLog runLog = new RunLog();
    runLog.setLogText(TEST_DESCRIPTION);
    runLog.setStatus(TaskStatus.SUCCESS);
    runLog.setTask(new PipelineTask(TEST_NAME));
    pipelineRunWithLog.setPipelineRun(run);
    pipelineRunWithLog.setRunOwner(USER);
    pipelineRunWithLog.setRunLogs(Collections.singletonList(runLog));
    EntityContainer<PipelineRunWithLog> container = EntityContainer.<PipelineRunWithLog>builder().entity(pipelineRunWithLog).owner(USER).permissions(PERMISSIONS_CONTAINER).build();
    XContentBuilder contentBuilder = mapper.map(container);
    verifyPipelineRun(run, TEST_NAME + " " + TEST_VERSION, contentBuilder);
    verifyRunParameters(Collections.singletonList(TEST_NAME + " " + TEST_VALUE), contentBuilder);
    verifyRunLogs(Collections.singletonList(TEST_NAME + " " + TEST_DESCRIPTION), contentBuilder);
    verifyPipelineUser(USER, contentBuilder);
    verifyPermissions(PERMISSIONS_CONTAINER, contentBuilder);
}