Search in sources :

Example 1 with Clusters

use of com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters in project platinum by hartwigmedical.

the class KubernetesEngine method findOrCreate.

public KubernetesCluster findOrCreate(final String runName, final List<TumorNormalPair> pairs, final JsonKey jsonKey, final String outputBucketName, final String serviceAccountEmail) {
    try {
        String clusterName = configuration.cluster().orElse(runName);
        GcpConfiguration gcpConfiguration = configuration.gcp();
        String parent = String.format("projects/%s/locations/%s", gcpConfiguration.projectOrThrow(), gcpConfiguration.regionOrThrow());
        if (find(fullPath(gcpConfiguration.projectOrThrow(), gcpConfiguration.regionOrThrow(), clusterName)).isEmpty()) {
            create(containerApi, parent, clusterName, gcpConfiguration);
        }
        if (!configuration.inCluster()) {
            if (!processRunner.execute(of("gcloud", "container", "clusters", "get-credentials", clusterName, "--region", gcpConfiguration.regionOrThrow(), "--project", gcpConfiguration.projectOrThrow()))) {
                throw new RuntimeException("Failed to get credentials for cluster");
            }
            if (!processRunner.execute(of("kubectl", "get", "configmaps"))) {
                throw new RuntimeException("Failed to run kubectl command against cluster");
            }
            LOGGER.info("Connection to cluster {} configured via gcloud and kubectl", Console.bold(clusterName));
        }
        DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient();
        TargetNodePool targetNodePool = configuration.gcp().nodePoolConfiguration().map(c -> TargetNodePool.fromConfig(c, configuration.batch().map(BatchConfiguration::size).orElse(configuration.samples().isEmpty() ? configuration.sampleIds().size() : configuration.samples().size()))).orElse(TargetNodePool.defaultPool());
        if (!targetNodePool.isDefault()) {
            new GcloudNodePool(processRunner).create(targetNodePool, serviceAccountEmail, clusterName, gcpConfiguration.projectOrThrow());
        }
        return new KubernetesCluster(runName, new JobScheduler(kubernetesClient, configuration.retryFailed()), new PipelineServiceAccountSecretVolume(jsonKey, kubernetesClient, "service-account-key"), new PipelineConfigMapVolume(pairs, kubernetesClient, runName), outputBucketName, serviceAccountEmail, configuration, Delay.threadSleep(), targetNodePool);
    } catch (Exception e) {
        throw new RuntimeException("Failed to create cluster", e);
    }
}
Also used : IPAllocationPolicy(com.google.api.services.container.v1beta1.model.IPAllocationPolicy) NodePool(com.google.api.services.container.v1beta1.model.NodePool) LoggerFactory(org.slf4j.LoggerFactory) NodeConfig(com.google.api.services.container.v1beta1.model.NodeConfig) Duration.ofSeconds(java.time.Duration.ofSeconds) Container(com.google.api.services.container.v1beta1.Container) JsonKey(com.hartwig.platinum.iam.JsonKey) List.of(java.util.List.of) GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) Operation(com.google.api.services.container.v1beta1.model.Operation) BatchConfiguration(com.hartwig.platinum.config.BatchConfiguration) DefaultKubernetesClient(io.fabric8.kubernetes.client.DefaultKubernetesClient) GcpConfiguration(com.hartwig.platinum.config.GcpConfiguration) TumorNormalPair(com.hartwig.platinum.p5sample.TumorNormalPair) PlatinumConfiguration(com.hartwig.platinum.config.PlatinumConfiguration) Console(com.hartwig.platinum.Console) Logger(org.slf4j.Logger) IOException(java.io.IOException) RetryPolicy(net.jodah.failsafe.RetryPolicy) Cluster(com.google.api.services.container.v1beta1.model.Cluster) Failsafe(net.jodah.failsafe.Failsafe) Create(com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters.Create) List(java.util.List) Get(com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters.Get) CreateClusterRequest(com.google.api.services.container.v1beta1.model.CreateClusterRequest) PrivateClusterConfig(com.google.api.services.container.v1beta1.model.PrivateClusterConfig) Duration.ofMinutes(java.time.Duration.ofMinutes) Optional(java.util.Optional) BatchConfiguration(com.hartwig.platinum.config.BatchConfiguration) GcpConfiguration(com.hartwig.platinum.config.GcpConfiguration) GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) IOException(java.io.IOException) DefaultKubernetesClient(io.fabric8.kubernetes.client.DefaultKubernetesClient)

Example 2 with Clusters

use of com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters in project platinum by hartwigmedical.

the class KubernetesEngine method create.

private static void create(final Container containerApi, final String parent, final String cluster, final GcpConfiguration gcpConfiguration) {
    try {
        Cluster newCluster = new Cluster();
        newCluster.setName(cluster);
        newCluster.setNetwork(gcpConfiguration.networkUrl());
        newCluster.setSubnetwork(gcpConfiguration.subnetUrl());
        newCluster.setLocations(gcpConfiguration.zones());
        NodePool defaultNodePool = new NodePool().setName("default").setInitialNodeCount(2);
        final NodeConfig nodeConfig = new NodeConfig().setPreemptible(gcpConfiguration.preemptibleCluster()).setOauthScopes(List.of("https://www.googleapis.com/auth/cloud-platform")).setDiskSizeGb(500);
        if (!gcpConfiguration.networkTags().isEmpty()) {
            nodeConfig.setTags(gcpConfiguration.networkTags());
        }
        defaultNodePool.setConfig(nodeConfig);
        newCluster.setNodePools(List.of(defaultNodePool));
        IPAllocationPolicy ipAllocationPolicy = new IPAllocationPolicy();
        if (gcpConfiguration.privateCluster()) {
            PrivateClusterConfig privateClusterConfig = new PrivateClusterConfig();
            privateClusterConfig.setEnablePrivateEndpoint(true);
            privateClusterConfig.setEnablePrivateNodes(true);
            privateClusterConfig.setMasterIpv4CidrBlock(gcpConfiguration.masterIpv4CidrBlock());
            newCluster.setPrivateCluster(true);
            newCluster.setPrivateClusterConfig(privateClusterConfig);
            ipAllocationPolicy.setUseIpAliases(true);
        }
        if (gcpConfiguration.secondaryRangeNamePods().isPresent() && gcpConfiguration.secondaryRangeNameServices().isPresent()) {
            ipAllocationPolicy.setClusterSecondaryRangeName(gcpConfiguration.secondaryRangeNamePods().get());
            ipAllocationPolicy.setServicesSecondaryRangeName(gcpConfiguration.secondaryRangeNameServices().get());
        }
        newCluster.setIpAllocationPolicy(ipAllocationPolicy);
        CreateClusterRequest createRequest = new CreateClusterRequest();
        createRequest.setCluster(newCluster);
        Create created = containerApi.projects().locations().clusters().create(parent, createRequest);
        Operation execute = created.execute();
        LOGGER.info("Creating new kubernetes cluster {} in project {} and region {}, this can take upwards of 5 minutes...", Console.bold(newCluster.getName()), Console.bold(gcpConfiguration.projectOrThrow()), Console.bold(gcpConfiguration.regionOrThrow()));
        Failsafe.with(new RetryPolicy<>().withMaxDuration(ofMinutes(15)).withDelay(ofSeconds(15)).withMaxAttempts(-1).handleResult(null).handleResult("RUNNING")).onFailure(objectExecutionCompletedEvent -> LOGGER.info("Waiting on operation, status is [{}]", objectExecutionCompletedEvent.getResult())).get(() -> containerApi.projects().locations().operations().get(String.format("projects/%s/locations/%s/operations/%s", gcpConfiguration.projectOrThrow(), gcpConfiguration.regionOrThrow(), execute.getName())).execute().getStatus());
    } catch (Exception e) {
        throw new RuntimeException("Failed to create cluster", e);
    }
}
Also used : IPAllocationPolicy(com.google.api.services.container.v1beta1.model.IPAllocationPolicy) NodePool(com.google.api.services.container.v1beta1.model.NodePool) LoggerFactory(org.slf4j.LoggerFactory) NodeConfig(com.google.api.services.container.v1beta1.model.NodeConfig) Duration.ofSeconds(java.time.Duration.ofSeconds) Container(com.google.api.services.container.v1beta1.Container) JsonKey(com.hartwig.platinum.iam.JsonKey) List.of(java.util.List.of) GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) Operation(com.google.api.services.container.v1beta1.model.Operation) BatchConfiguration(com.hartwig.platinum.config.BatchConfiguration) DefaultKubernetesClient(io.fabric8.kubernetes.client.DefaultKubernetesClient) GcpConfiguration(com.hartwig.platinum.config.GcpConfiguration) TumorNormalPair(com.hartwig.platinum.p5sample.TumorNormalPair) PlatinumConfiguration(com.hartwig.platinum.config.PlatinumConfiguration) Console(com.hartwig.platinum.Console) Logger(org.slf4j.Logger) IOException(java.io.IOException) RetryPolicy(net.jodah.failsafe.RetryPolicy) Cluster(com.google.api.services.container.v1beta1.model.Cluster) Failsafe(net.jodah.failsafe.Failsafe) Create(com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters.Create) List(java.util.List) Get(com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters.Get) CreateClusterRequest(com.google.api.services.container.v1beta1.model.CreateClusterRequest) PrivateClusterConfig(com.google.api.services.container.v1beta1.model.PrivateClusterConfig) Duration.ofMinutes(java.time.Duration.ofMinutes) Optional(java.util.Optional) CreateClusterRequest(com.google.api.services.container.v1beta1.model.CreateClusterRequest) PrivateClusterConfig(com.google.api.services.container.v1beta1.model.PrivateClusterConfig) IPAllocationPolicy(com.google.api.services.container.v1beta1.model.IPAllocationPolicy) Create(com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters.Create) NodePool(com.google.api.services.container.v1beta1.model.NodePool) Cluster(com.google.api.services.container.v1beta1.model.Cluster) Operation(com.google.api.services.container.v1beta1.model.Operation) RetryPolicy(net.jodah.failsafe.RetryPolicy) GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) IOException(java.io.IOException) NodeConfig(com.google.api.services.container.v1beta1.model.NodeConfig)

Example 3 with Clusters

use of com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters in project platinum by hartwigmedical.

the class KubernetesEngineTest method setup.

@Before
public void setup() {
    final Projects projects = mock(Projects.class);
    processRunner = mock(ProcessRunner.class);
    final Container container = mock(Container.class);
    locations = mock(Locations.class);
    clusters = mock(Clusters.class);
    when(container.projects()).thenReturn(projects);
    when(projects.locations()).thenReturn(locations);
    when(locations.clusters()).thenReturn(clusters);
    when(processRunner.execute(anyList())).thenReturn(true);
    victim = new KubernetesEngine(container, processRunner, CONFIGURATION);
}
Also used : Container(com.google.api.services.container.v1beta1.Container) Locations(com.google.api.services.container.v1beta1.Container.Projects.Locations) Clusters(com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters) Projects(com.google.api.services.container.v1beta1.Container.Projects) Before(org.junit.Before)

Aggregations

Container (com.google.api.services.container.v1beta1.Container)3 GoogleJsonResponseException (com.google.api.client.googleapis.json.GoogleJsonResponseException)2 Create (com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters.Create)2 Get (com.google.api.services.container.v1beta1.Container.Projects.Locations.Clusters.Get)2 Cluster (com.google.api.services.container.v1beta1.model.Cluster)2 CreateClusterRequest (com.google.api.services.container.v1beta1.model.CreateClusterRequest)2 IPAllocationPolicy (com.google.api.services.container.v1beta1.model.IPAllocationPolicy)2 NodeConfig (com.google.api.services.container.v1beta1.model.NodeConfig)2 NodePool (com.google.api.services.container.v1beta1.model.NodePool)2 Operation (com.google.api.services.container.v1beta1.model.Operation)2 PrivateClusterConfig (com.google.api.services.container.v1beta1.model.PrivateClusterConfig)2 Console (com.hartwig.platinum.Console)2 BatchConfiguration (com.hartwig.platinum.config.BatchConfiguration)2 GcpConfiguration (com.hartwig.platinum.config.GcpConfiguration)2 PlatinumConfiguration (com.hartwig.platinum.config.PlatinumConfiguration)2 JsonKey (com.hartwig.platinum.iam.JsonKey)2 TumorNormalPair (com.hartwig.platinum.p5sample.TumorNormalPair)2 DefaultKubernetesClient (io.fabric8.kubernetes.client.DefaultKubernetesClient)2 IOException (java.io.IOException)2 Duration.ofMinutes (java.time.Duration.ofMinutes)2