Examples with InstanceType com.epam.pipeline.entity.cluster.InstanceType used on opensource projects

Example 1 with InstanceType

use of com.epam.pipeline.entity.cluster.InstanceType in project cloud-pipeline by epam.

the class InstanceOfferManagerUnitTest method getAllowedInstanceAndPriceTypesShouldSearchPreferencesUsingToolAsExternalResourceIfToolIdIsSpecified.

@Test
public void getAllowedInstanceAndPriceTypesShouldSearchPreferencesUsingToolAsExternalResourceIfToolIdIsSpecified() {
    final List<String> allowedPriceTypes = Arrays.asList(SPOT, ON_DEMAND);
    final List<InstanceType> allInstanceTypes = Arrays.asList(m4InstanceType, m5InstanceType, t2InstanceType);
    final List<InstanceType> allowedInstanceTypes = Arrays.asList(m4InstanceType, m5InstanceType);
    final List<InstanceType> allowedInstanceDockerTypes = Collections.singletonList(m4InstanceType);
    when(contextualPreferenceManager.search(eq(INSTANCE_TYPES_PREFERENCES), eq(TOOL_RESOURCE))).thenReturn(new ContextualPreference(ALLOWED_INSTANCE_TYPES_PREFERENCE, M4_M5_PATTERNS));
    when(contextualPreferenceManager.search(eq(Arrays.asList(ALLOWED_DOCKER_INSTANCE_TYPES_PREFERENCE, ALLOWED_INSTANCE_TYPES_PREFERENCE)), eq(TOOL_RESOURCE))).thenReturn(new ContextualPreference(ALLOWED_DOCKER_INSTANCE_TYPES_PREFERENCE, M4_PATTERN));
    when(contextualPreferenceManager.search(eq(Collections.singletonList(ALLOWED_PRICE_TYPES_PREFERENCE)), eq(TOOL_RESOURCE))).thenReturn(new ContextualPreference(ALLOWED_PRICE_TYPES_PREFERENCE, SPOT_AND_ON_DEMAND_TYPES));
    when(instanceOfferDao.loadInstanceTypes(any())).thenReturn(allInstanceTypes);
    final AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = instanceOfferManager.getAllowedInstanceAndPriceTypes(TOOL_ID);
    assertThat(allowedInstanceAndPriceTypes.getAllowedInstanceTypes(), is(allowedInstanceTypes));
    assertThat(allowedInstanceAndPriceTypes.getAllowedInstanceDockerTypes(), is(allowedInstanceDockerTypes));
    assertThat(allowedInstanceAndPriceTypes.getAllowedPriceTypes(), is(allowedPriceTypes));
}

Example 2 with InstanceType

use of com.epam.pipeline.entity.cluster.InstanceType in project cloud-pipeline by epam.

the class ResourceMonitoringManager method processRuns.

private List<PipelineRun> processRuns(Map<String, PipelineRun> running, Map<String, Double> cpuMetrics, double idleCpuLevel, int actionTimeout, IdleRunAction action) {
    List<PipelineRun> runsToUpdate = new ArrayList<>(running.size());
    List<Pair<PipelineRun, Double>> runsToNotify = new ArrayList<>(running.size());
    for (Map.Entry<String, PipelineRun> entry : running.entrySet()) {
        PipelineRun run = entry.getValue();
        if (run.isNonPause()) {
            continue;
        }
        Double metric = cpuMetrics.get(entry.getKey());
        if (metric != null) {
            InstanceType type = instanceTypeMap.getOrDefault(run.getInstance().getNodeType(), InstanceType.builder().vCPU(1).build());
            double cpuUsageRate = metric / MILLIS / type.getVCPU();
            if (Precision.compareTo(cpuUsageRate, idleCpuLevel, ONE_THOUSANDTH) < 0) {
                processIdleRun(run, actionTimeout, action, runsToNotify, runsToUpdate, cpuUsageRate);
            } else if (run.getLastIdleNotificationTime() != null) {
                // No action is longer needed, clear timeout
                run.setLastIdleNotificationTime(null);
                runsToUpdate.add(run);
            }
        }
    }
    notificationManager.notifyIdleRuns(runsToNotify, NotificationType.IDLE_RUN);
    return runsToUpdate;
}

Example 3 with InstanceType

use of com.epam.pipeline.entity.cluster.InstanceType in project cloud-pipeline by epam.

the class ToolManagerTest method setUp.

@Before
public void setUp() {
    MockitoAnnotations.initMocks(this);
    TestUtils.configureDockerClientMock(dockerClient, dockerClientFactory);
    firstRegistry = new DockerRegistry();
    firstRegistry.setPath(TEST_REPO);
    firstRegistry.setOwner(TEST_USER);
    registryDao.createDockerRegistry(firstRegistry);
    secondRegistry = new DockerRegistry();
    secondRegistry.setPath(TEST_REPO_2);
    secondRegistry.setOwner(TEST_USER);
    registryDao.createDockerRegistry(secondRegistry);
    LABELS.add(LABEL_1);
    LABELS.add(LABEL_2);
    ENDPOINTS.add("{\"nginx\" : {\"port\" : 8080}}");
    ENDPOINTS.add("9080");
    firstToolGroup = new ToolGroup();
    firstToolGroup.setName(TEST_GROUP_NAME);
    firstToolGroup.setRegistryId(firstRegistry.getId());
    toolGroupManager.create(firstToolGroup);
    secondToolGroup = new ToolGroup();
    secondToolGroup.setName(TEST_GROUP_NAME);
    secondToolGroup.setRegistryId(secondRegistry.getId());
    toolGroupManager.create(secondToolGroup);
    ToolVersion toolVersion = new ToolVersion();
    toolVersion.setDigest("test_digest");
    toolVersion.setSize(DOCKER_SIZE);
    toolVersion.setVersion("test_version");
    Mockito.when(dockerClient.getVersionAttributes(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(toolVersion);
    final InstanceType instanceType = new InstanceType();
    instanceType.setName(TEST_ALLOWED_INSTANCE_TYPE);
    instanceOfferManager.updateOfferedInstanceTypes(Collections.singletonList(instanceType));
}

Example 4 with InstanceType

use of com.epam.pipeline.entity.cluster.InstanceType in project cloud-pipeline by epam.

the class InstanceOfferManager method getAllowedInstanceAndPriceTypes.

/**
 * Returns allowed instance and price types for a current user.
 *
 * @param toolId Optional tool id. If specified than allowed instance and price types will be bounded for a
 *               specific tool.
 */
public AllowedInstanceAndPriceTypes getAllowedInstanceAndPriceTypes(final String toolId) {
    final ContextualPreferenceExternalResource resource = toolId != null ? new ContextualPreferenceExternalResource(ContextualPreferenceLevel.TOOL, toolId) : null;
    final List<InstanceType> instanceTypes = getAllInstanceTypes();
    final List<InstanceType> allowedInstanceTypes = getAllowedInstanceTypes(instanceTypes, resource, SystemPreferences.CLUSTER_ALLOWED_INSTANCE_TYPES);
    final List<InstanceType> allowedInstanceDockerTypes = getAllowedInstanceTypes(instanceTypes, resource, SystemPreferences.CLUSTER_ALLOWED_INSTANCE_TYPES_DOCKER, SystemPreferences.CLUSTER_ALLOWED_INSTANCE_TYPES);
    final List<String> allowedPriceTypes = getContextualPreferenceValueAsList(resource, SystemPreferences.CLUSTER_ALLOWED_PRICE_TYPES);
    return new AllowedInstanceAndPriceTypes(allowedInstanceTypes, allowedInstanceDockerTypes, allowedPriceTypes);
}

Example 5 with InstanceType

use of com.epam.pipeline.entity.cluster.InstanceType in project cloud-pipeline by epam.

the class InstanceOfferManager method getInstanceEstimatedPrice.

public InstancePrice getInstanceEstimatedPrice(Long id, String version, String configName, String instanceType, int instanceDisk, Boolean spot, Long regionId) throws GitClientException {
    Boolean useSpot = spot;
    if (StringUtils.isEmpty(instanceType) || instanceDisk <= 0 || useSpot == null) {
        PipelineConfiguration pipelineConfiguration = versionManager.loadParametersFromScript(id, version, configName);
        if (StringUtils.isEmpty(instanceType)) {
            instanceType = pipelineConfiguration.getInstanceType();
        }
        if (instanceDisk <= 0) {
            instanceDisk = Integer.parseInt(pipelineConfiguration.getInstanceDisk());
        }
        if (useSpot == null) {
            useSpot = pipelineConfiguration.getIsSpot();
        }
    }
    Assert.isTrue(isInstanceAllowed(instanceType), messageHelper.getMessage(MessageConstants.ERROR_INSTANCE_TYPE_IS_NOT_ALLOWED, instanceType));
    AwsRegion region = awsRegionManager.loadRegionOrGetDefault(regionId);
    double pricePerHourForInstance = getPricePerHourForInstance(instanceType, isSpotRequest(useSpot), region.getAwsRegionName());
    double pricePerDisk = getPriceForDisk(instanceDisk, region.getAwsRegionName());
    double pricePerHour = pricePerDisk + pricePerHourForInstance;
    InstancePrice instancePrice = new InstancePrice(instanceType, instanceDisk, pricePerHour);
    List<PipelineRun> runs = pipelineRunManager.loadAllRunsByPipeline(id, version).stream().filter(run -> run.getStatus().isFinal()).collect(Collectors.toList());
    if (!runs.isEmpty()) {
        long minimumDuration = -1;
        long maximumDuration = -1;
        long totalDurations = 0;
        for (PipelineRun run : runs) {
            long duration = run.getEndDate().getTime() - run.getStartDate().getTime();
            if (minimumDuration == -1 || minimumDuration > duration) {
                minimumDuration = duration;
            }
            if (maximumDuration == -1 || maximumDuration < duration) {
                maximumDuration = duration;
            }
            totalDurations += duration;
        }
        double averageDuration = (double) totalDurations / runs.size();
        instancePrice.setAverageTimePrice(pricePerHour * averageDuration / ONE_HOUR);
        instancePrice.setMinimumTimePrice(pricePerHour * minimumDuration / ONE_HOUR);
        instancePrice.setMaximumTimePrice(pricePerHour * maximumDuration / ONE_HOUR);
    }
    return instancePrice;
}