Examples with GcpConfiguration com.hartwig.platinum.config.GcpConfiguration used on opensource projects

Example 1 with GcpConfiguration

use of com.hartwig.platinum.config.GcpConfiguration in project platinum by hartwigmedical.

the class KubernetesEngine method findOrCreate.

public KubernetesCluster findOrCreate(final String runName, final List<TumorNormalPair> pairs, final JsonKey jsonKey, final String outputBucketName, final String serviceAccountEmail) {
    try {
        String clusterName = configuration.cluster().orElse(runName);
        GcpConfiguration gcpConfiguration = configuration.gcp();
        String parent = String.format("projects/%s/locations/%s", gcpConfiguration.projectOrThrow(), gcpConfiguration.regionOrThrow());
        if (find(fullPath(gcpConfiguration.projectOrThrow(), gcpConfiguration.regionOrThrow(), clusterName)).isEmpty()) {
            create(containerApi, parent, clusterName, gcpConfiguration);
        }
        if (!configuration.inCluster()) {
            if (!processRunner.execute(of("gcloud", "container", "clusters", "get-credentials", clusterName, "--region", gcpConfiguration.regionOrThrow(), "--project", gcpConfiguration.projectOrThrow()))) {
                throw new RuntimeException("Failed to get credentials for cluster");
            }
            if (!processRunner.execute(of("kubectl", "get", "configmaps"))) {
                throw new RuntimeException("Failed to run kubectl command against cluster");
            }
            LOGGER.info("Connection to cluster {} configured via gcloud and kubectl", Console.bold(clusterName));
        }
        DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient();
        TargetNodePool targetNodePool = configuration.gcp().nodePoolConfiguration().map(c -> TargetNodePool.fromConfig(c, configuration.batch().map(BatchConfiguration::size).orElse(configuration.samples().isEmpty() ? configuration.sampleIds().size() : configuration.samples().size()))).orElse(TargetNodePool.defaultPool());
        if (!targetNodePool.isDefault()) {
            new GcloudNodePool(processRunner).create(targetNodePool, serviceAccountEmail, clusterName, gcpConfiguration.projectOrThrow());
        }
        return new KubernetesCluster(runName, new JobScheduler(kubernetesClient, configuration.retryFailed()), new PipelineServiceAccountSecretVolume(jsonKey, kubernetesClient, "service-account-key"), new PipelineConfigMapVolume(pairs, kubernetesClient, runName), outputBucketName, serviceAccountEmail, configuration, Delay.threadSleep(), targetNodePool);
    } catch (Exception e) {
        throw new RuntimeException("Failed to create cluster", e);
    }
}

Example 2 with GcpConfiguration

use of com.hartwig.platinum.config.GcpConfiguration in project platinum by hartwigmedical.

the class KubernetesEngine method create.

private static void create(final Container containerApi, final String parent, final String cluster, final GcpConfiguration gcpConfiguration) {
    try {
        Cluster newCluster = new Cluster();
        newCluster.setName(cluster);
        newCluster.setNetwork(gcpConfiguration.networkUrl());
        newCluster.setSubnetwork(gcpConfiguration.subnetUrl());
        newCluster.setLocations(gcpConfiguration.zones());
        NodePool defaultNodePool = new NodePool().setName("default").setInitialNodeCount(2);
        final NodeConfig nodeConfig = new NodeConfig().setPreemptible(gcpConfiguration.preemptibleCluster()).setOauthScopes(List.of("https://www.googleapis.com/auth/cloud-platform")).setDiskSizeGb(500);
        if (!gcpConfiguration.networkTags().isEmpty()) {
            nodeConfig.setTags(gcpConfiguration.networkTags());
        }
        defaultNodePool.setConfig(nodeConfig);
        newCluster.setNodePools(List.of(defaultNodePool));
        IPAllocationPolicy ipAllocationPolicy = new IPAllocationPolicy();
        if (gcpConfiguration.privateCluster()) {
            PrivateClusterConfig privateClusterConfig = new PrivateClusterConfig();
            privateClusterConfig.setEnablePrivateEndpoint(true);
            privateClusterConfig.setEnablePrivateNodes(true);
            privateClusterConfig.setMasterIpv4CidrBlock(gcpConfiguration.masterIpv4CidrBlock());
            newCluster.setPrivateCluster(true);
            newCluster.setPrivateClusterConfig(privateClusterConfig);
            ipAllocationPolicy.setUseIpAliases(true);
        }
        if (gcpConfiguration.secondaryRangeNamePods().isPresent() && gcpConfiguration.secondaryRangeNameServices().isPresent()) {
            ipAllocationPolicy.setClusterSecondaryRangeName(gcpConfiguration.secondaryRangeNamePods().get());
            ipAllocationPolicy.setServicesSecondaryRangeName(gcpConfiguration.secondaryRangeNameServices().get());
        }
        newCluster.setIpAllocationPolicy(ipAllocationPolicy);
        CreateClusterRequest createRequest = new CreateClusterRequest();
        createRequest.setCluster(newCluster);
        Create created = containerApi.projects().locations().clusters().create(parent, createRequest);
        Operation execute = created.execute();
        LOGGER.info("Creating new kubernetes cluster {} in project {} and region {}, this can take upwards of 5 minutes...", Console.bold(newCluster.getName()), Console.bold(gcpConfiguration.projectOrThrow()), Console.bold(gcpConfiguration.regionOrThrow()));
        Failsafe.with(new RetryPolicy<>().withMaxDuration(ofMinutes(15)).withDelay(ofSeconds(15)).withMaxAttempts(-1).handleResult(null).handleResult("RUNNING")).onFailure(objectExecutionCompletedEvent -> LOGGER.info("Waiting on operation, status is [{}]", objectExecutionCompletedEvent.getResult())).get(() -> containerApi.projects().locations().operations().get(String.format("projects/%s/locations/%s/operations/%s", gcpConfiguration.projectOrThrow(), gcpConfiguration.regionOrThrow(), execute.getName())).execute().getStatus());
    } catch (Exception e) {
        throw new RuntimeException("Failed to create cluster", e);
    }
}

Example 3 with GcpConfiguration

use of com.hartwig.platinum.config.GcpConfiguration in project platinum by hartwigmedical.

the class PipelineArguments method fixed.

private Map<String, String> fixed(final String secretsPath, final String serviceAccountKeySecretName) {
    GcpConfiguration gcpConfiguration = platinumConfiguration.gcp();
    ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String>builder().put("-output_bucket", outputBucket).put("-private_key_path", format("%s/%s", secretsPath, serviceAccountKeySecretName)).put("-project", gcpConfiguration.projectOrThrow()).put("-region", gcpConfiguration.regionOrThrow()).put("-network", gcpConfiguration.networkUrl()).put("-subnet", gcpConfiguration.subnetUrl()).put("-service_account_email", serviceAccountEmail);
    if (!gcpConfiguration.networkTags().isEmpty()) {
        builder.put("-network_tags", String.join(",", gcpConfiguration.networkTags()));
    }
    if (platinumConfiguration.apiUrl().isPresent()) {
        builder.put("-profile", "production");
        builder.put("-context", "RESEARCH");
    } else {
        builder.put("-context", "PLATINUM");
    }
    if (platinumConfiguration.cmek().isPresent()) {
        builder.put("-cmek", platinumConfiguration.cmek().get());
    }
    return builder.build();
}

Example 4 with GcpConfiguration

use of com.hartwig.platinum.config.GcpConfiguration in project platinum by hartwigmedical.

the class Platinum method run.

public void run() {
    LOGGER.info("Starting platinum run with name {} and input {}", Console.bold(runName), Console.bold(input));
    GcpConfiguration gcpConfiguration = configuration.gcp();
    PipelineServiceAccount serviceAccount = PipelineServiceAccount.from(iam, resourceManager, runName, gcpConfiguration.projectOrThrow(), configuration);
    String serviceAccountEmail = serviceAccount.findOrCreate();
    ServiceAccountPrivateKey privateKey = ServiceAccountPrivateKey.from(configuration, iam);
    JsonKey jsonKey = privateKey.create(gcpConfiguration.projectOrThrow(), serviceAccountEmail);
    List<TumorNormalPair> pairs = DecomposeSamples.apply(configuration.sampleBucket().map(b -> new SampleBucket(storage.get(b)).apply()).orElseGet(configuration::samples));
    int submitted = kubernetesEngine.findOrCreate(runName, pairs, jsonKey, OutputBucket.from(storage).findOrCreate(runName, gcpConfiguration.regionOrThrow(), serviceAccountEmail, configuration), serviceAccountEmail).submit(samples(configuration, pairs, runName));
    LOGGER.info("Platinum started {} pipelines on GCP", Console.bold(String.valueOf(submitted)));
    LOGGER.info("You can monitor their progress with: {}", Console.bold("./platinum status"));
}