Examples with ClusterOperationMetadata com.google.cloud.dataproc.v1.ClusterOperationMetadata used on opensource projects

Example 1 with ClusterOperationMetadata

use of com.google.cloud.dataproc.v1.ClusterOperationMetadata in project cdap by caskdata.

the class DataprocProvisioner method createCluster.

@Override
public Cluster createCluster(ProvisionerContext context) throws Exception {
    DataprocConf conf = DataprocConf.create(createContextProperties(context));
    if (!isAutoscalingFieldsValid(conf, createContextProperties(context))) {
        LOG.warn("The configs : {}, {}, {} will not be considered when {} is enabled ", DataprocConf.WORKER_NUM_NODES, DataprocConf.SECONDARY_WORKER_NUM_NODES, DataprocConf.AUTOSCALING_POLICY, DataprocConf.PREDEFINED_AUTOSCALE_ENABLED);
    }
    if (context.getRuntimeMonitorType() == RuntimeMonitorType.SSH || !conf.isRuntimeJobManagerEnabled()) {
        // Generates and set the ssh key if it does not have one.
        // Since invocation of this method can come from a retry, we don't need to keep regenerating the keys
        SSHContext sshContext = context.getSSHContext();
        if (sshContext != null) {
            SSHKeyPair sshKeyPair = sshContext.getSSHKeyPair().orElse(null);
            if (sshKeyPair == null) {
                sshKeyPair = sshContext.generate("cdap");
                sshContext.setSSHKeyPair(sshKeyPair);
            }
            conf = DataprocConf.create(createContextProperties(context), sshKeyPair.getPublicKey());
        }
    }
    try (DataprocClient client = getClient(conf)) {
        Cluster reused = tryReuseCluster(client, context, conf);
        if (reused != null) {
            DataprocUtils.emitMetric(context, conf.getRegion(), "provisioner.createCluster.reuse.count");
            return reused;
        }
        String clusterName = getRunKey(context);
        // if it already exists, it means this is a retry. We can skip actually making the request
        Optional<Cluster> existing = client.getCluster(clusterName);
        if (existing.isPresent()) {
            return existing.get();
        }
        String imageVersion = getImageVersion(context, conf);
        String imageDescription = conf.getCustomImageUri();
        if (imageDescription == null || imageDescription.isEmpty()) {
            imageDescription = imageVersion;
        }
        // Reload system context properties and get system labels
        Map<String, String> labels = new HashMap<>();
        labels.putAll(getSystemLabels());
        labels.putAll(getReuseLabels(context, conf));
        labels.putAll(conf.getClusterLabels());
        LOG.info("Creating Dataproc cluster {} in project {}, in region {}, with image {}, with labels {}, endpoint {}", clusterName, conf.getProjectId(), conf.getRegion(), imageDescription, labels, getRootUrl(conf));
        boolean privateInstance = Boolean.parseBoolean(getSystemContext().getProperties().get(PRIVATE_INSTANCE));
        ClusterOperationMetadata createOperationMeta = client.createCluster(clusterName, imageVersion, labels, privateInstance);
        int numWarnings = createOperationMeta.getWarningsCount();
        if (numWarnings > 0) {
            LOG.warn("Encountered {} warning{} while creating Dataproc cluster:\n{}", numWarnings, numWarnings > 1 ? "s" : "", String.join("\n", createOperationMeta.getWarningsList()));
        }
        DataprocUtils.emitMetric(context, conf.getRegion(), "provisioner.createCluster.response.count");
        return new Cluster(clusterName, ClusterStatus.CREATING, Collections.emptyList(), Collections.emptyMap());
    } catch (Exception e) {
        DataprocUtils.emitMetric(context, conf.getRegion(), "provisioner.createCluster.response.count", e);
        throw e;
    }
}

Example 2 with ClusterOperationMetadata

use of com.google.cloud.dataproc.v1.ClusterOperationMetadata in project cdap by caskdata.

the class DataprocClient method updateClusterLabels.

/**
 * Updates labelsToSet on the given Dataproc cluster.
 *
 * @param clusterName name of the cluster
 * @param labelsToSet Key/Value pairs to set on the Dataproc cluster.
 * @param labelsToRemove collection of labels to remove from the Dataproc cluster.
 */
void updateClusterLabels(String clusterName, Map<String, String> labelsToSet, Collection<String> labelsToRemove) throws RetryableProvisionException, InterruptedException {
    if (labelsToSet.isEmpty() && labelsToRemove.isEmpty()) {
        return;
    }
    try {
        Cluster cluster = getDataprocCluster(clusterName).filter(c -> c.getStatus().getState() == ClusterStatus.State.RUNNING).orElseThrow(() -> new DataprocRuntimeException("Dataproc cluster " + clusterName + " does not exist or not in running state"));
        Map<String, String> existingLabels = cluster.getLabelsMap();
        // no need to update the cluster labelsToSet.
        if (labelsToSet.entrySet().stream().allMatch(e -> Objects.equals(e.getValue(), existingLabels.get(e.getKey()))) && labelsToRemove.stream().noneMatch(existingLabels::containsKey)) {
            return;
        }
        Map<String, String> newLabels = new HashMap<>(existingLabels);
        newLabels.keySet().removeAll(labelsToRemove);
        newLabels.putAll(labelsToSet);
        FieldMask updateMask = FieldMask.newBuilder().addPaths("labels").build();
        OperationFuture<Cluster, ClusterOperationMetadata> operationFuture = client.updateClusterAsync(UpdateClusterRequest.newBuilder().setProjectId(conf.getProjectId()).setRegion(conf.getRegion()).setClusterName(clusterName).setCluster(cluster.toBuilder().clearLabels().putAllLabels(newLabels)).setUpdateMask(updateMask).build());
        ClusterOperationMetadata metadata = operationFuture.getMetadata().get();
        int numWarnings = metadata.getWarningsCount();
        if (numWarnings > 0) {
            LOG.warn("Encountered {} warning {} while setting labels on cluster:\n{}", numWarnings, numWarnings > 1 ? "s" : "", String.join("\n", metadata.getWarningsList()));
        }
    } catch (ExecutionException e) {
        Throwable cause = e.getCause();
        if (cause instanceof ApiException) {
            throw handleApiException((ApiException) cause);
        }
        throw new DataprocRuntimeException(cause);
    }
}

Example 3 with ClusterOperationMetadata

use of com.google.cloud.dataproc.v1.ClusterOperationMetadata in project cdap by caskdata.

the class DataprocClient method createCluster.

/**
 * Create a cluster. This will return after the initial request to create the cluster is completed.
 * At this point, the cluster is likely not yet running, but in a provisioning state.
 *
 * @param name         the name of the cluster to create
 * @param imageVersion the image version for the cluster
 * @param labels       labels to set on the cluster
 * @param privateInstance {@code true} to indicate using private instance
 * @return create operation metadata
 * @throws InterruptedException        if the thread was interrupted while waiting for the initial request to complete
 * @throws AlreadyExistsException      if the cluster already exists
 * @throws IOException                 if there was an I/O error talking to Google Compute APIs
 * @throws RetryableProvisionException if there was a non 4xx error code returned
 */
ClusterOperationMetadata createCluster(String name, String imageVersion, Map<String, String> labels, boolean privateInstance) throws RetryableProvisionException, InterruptedException, IOException {
    if (network == null) {
        // yet being used to create cluster.
        throw new IllegalArgumentException("Missing network information");
    }
    try {
        Map<String, String> metadata = new HashMap<>();
        SSHPublicKey publicKey = conf.getPublicKey();
        if (publicKey != null) {
            // Don't fail if there is no public key. It is for tooling case that the key might be generated differently.
            metadata.put("ssh-keys", publicKey.getUser() + ":" + publicKey.getKey());
            // override any os-login that may be set on the project-level metadata
            // this metadata is only needed if ssh is being used to launch the jobs - CDAP-15369
            metadata.put("enable-oslogin", "false");
        }
        // Check if ClusterMetaData is provided and add them.
        metadata.putAll(conf.getClusterMetaData());
        GceClusterConfig.Builder clusterConfig = GceClusterConfig.newBuilder().addServiceAccountScopes(DataprocConf.CLOUD_PLATFORM_SCOPE).setShieldedInstanceConfig(ShieldedInstanceConfig.newBuilder().setEnableSecureBoot(conf.isSecureBootEnabled()).setEnableVtpm(conf.isvTpmEnabled()).setEnableIntegrityMonitoring(conf.isIntegrityMonitoringEnabled()).build()).putAllMetadata(metadata);
        if (conf.getServiceAccount() != null) {
            clusterConfig.setServiceAccount(conf.getServiceAccount());
        }
        if (conf.getZone() != null) {
            clusterConfig.setZoneUri(conf.getZone());
        }
        // subnets are unique within a location, not within a network, which is why these configs are mutually exclusive.
        if (conf.getSubnet() != null) {
            clusterConfig.setSubnetworkUri(conf.getSubnet());
        } else {
            clusterConfig.setNetworkUri(network.getSelfLink());
        }
        // Add any defined Network Tags
        clusterConfig.addAllTags(conf.getNetworkTags());
        boolean internalIPOnly = isInternalIPOnly(network, privateInstance, publicKey != null);
        // if public key is not null that means ssh is used to launch / monitor job on dataproc
        if (publicKey != null) {
            int maxTags = Math.max(0, DataprocConf.MAX_NETWORK_TAGS - clusterConfig.getTagsCount());
            List<String> tags = getFirewallTargetTags(network, internalIPOnly);
            if (tags.size() > maxTags) {
                LOG.warn("No more than 64 tags can be added. Firewall tags ignored: {}", tags.subList(maxTags, tags.size()));
            }
            tags.stream().limit(maxTags).forEach(clusterConfig::addTags);
        }
        // if internal ip is preferred then create dataproc cluster without external ip for better security
        clusterConfig.setInternalIpOnly(internalIPOnly);
        Map<String, String> clusterProperties = new HashMap<>(conf.getClusterProperties());
        // Enable/Disable stackdriver
        clusterProperties.put("dataproc:dataproc.logging.stackdriver.enable", Boolean.toString(conf.isStackdriverLoggingEnabled()));
        clusterProperties.put("dataproc:dataproc.monitoring.stackdriver.enable", Boolean.toString(conf.isStackdriverMonitoringEnabled()));
        DiskConfig workerDiskConfig = DiskConfig.newBuilder().setBootDiskSizeGb(conf.getWorkerDiskGB()).setBootDiskType(conf.getWorkerDiskType()).setNumLocalSsds(0).build();
        InstanceGroupConfig.Builder primaryWorkerConfig = InstanceGroupConfig.newBuilder().setNumInstances(conf.getWorkerNumNodes()).setMachineTypeUri(conf.getWorkerMachineType()).setDiskConfig(workerDiskConfig);
        InstanceGroupConfig.Builder secondaryWorkerConfig = InstanceGroupConfig.newBuilder().setNumInstances(conf.getSecondaryWorkerNumNodes()).setMachineTypeUri(conf.getWorkerMachineType()).setPreemptibility(InstanceGroupConfig.Preemptibility.NON_PREEMPTIBLE).setDiskConfig(workerDiskConfig);
        // Set default concurrency settings for fixed cluster
        if (Strings.isNullOrEmpty(conf.getAutoScalingPolicy()) && !conf.isPredefinedAutoScaleEnabled()) {
            // Set spark.default.parallelism according to cluster size.
            // Spark defaults it to number of current executors, but when we configure the job
            // executors may not have started yet, so this value gets artificially low.
            int defaultConcurrency = Math.max(conf.getTotalWorkerCPUs(), MIN_DEFAULT_CONCURRENCY);
            // Set spark.sql.adaptive.coalescePartitions.initialPartitionNum as 32x of default parallelism,
            // but no more than 8192. This value is used only in spark 3 with adaptive execution and
            // according to our tests spark can handle really large numbers and 32x is a reasonable default.
            int initialPartitionNum = Math.min(Math.max(conf.getTotalWorkerCPUs() * PARTITION_NUM_FACTOR, MIN_INITIAL_PARTITIONS_DEFAULT), MAX_INITIAL_PARTITIONS_DEFAULT);
            clusterProperties.putIfAbsent("spark:spark.default.parallelism", Integer.toString(defaultConcurrency));
            clusterProperties.putIfAbsent("spark:spark.sql.adaptive.coalescePartitions.initialPartitionNum", Integer.toString(initialPartitionNum));
        }
        SoftwareConfig.Builder softwareConfigBuilder = SoftwareConfig.newBuilder().putAllProperties(clusterProperties);
        // Use image version only if custom Image URI is not specified, otherwise may cause image version conflicts
        if (conf.getCustomImageUri() == null || conf.getCustomImageUri().isEmpty()) {
            softwareConfigBuilder.setImageVersion(imageVersion);
        } else {
            // If custom Image URI is specified, use that for cluster creation
            primaryWorkerConfig.setImageUri(conf.getCustomImageUri());
            secondaryWorkerConfig.setImageUri(conf.getCustomImageUri());
        }
        ClusterConfig.Builder builder = ClusterConfig.newBuilder().setEndpointConfig(EndpointConfig.newBuilder().setEnableHttpPortAccess(conf.isComponentGatewayEnabled()).build()).setMasterConfig(InstanceGroupConfig.newBuilder().setNumInstances(conf.getMasterNumNodes()).setMachineTypeUri(conf.getMasterMachineType()).setDiskConfig(DiskConfig.newBuilder().setBootDiskType(conf.getMasterDiskType()).setBootDiskSizeGb(conf.getMasterDiskGB()).setNumLocalSsds(0).build()).build()).setWorkerConfig(primaryWorkerConfig.build()).setSecondaryWorkerConfig(secondaryWorkerConfig.build()).setGceClusterConfig(clusterConfig.build()).setSoftwareConfig(softwareConfigBuilder);
        // Cluster TTL if one should be set
        if (conf.getIdleTTLMinutes() > 0) {
            long seconds = TimeUnit.MINUTES.toSeconds(conf.getIdleTTLMinutes());
            builder.setLifecycleConfig(LifecycleConfig.newBuilder().setIdleDeleteTtl(Duration.newBuilder().setSeconds(seconds).build()).build());
        }
        // Add any Node Initialization action scripts
        for (String action : conf.getInitActions()) {
            builder.addInitializationActions(NodeInitializationAction.newBuilder().setExecutableFile(action).build());
        }
        // Set Auto Scaling Policy
        String autoScalingPolicy = conf.getAutoScalingPolicy();
        if (conf.isPredefinedAutoScaleEnabled()) {
            PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(conf);
            autoScalingPolicy = predefinedAutoScaling.createPredefinedAutoScalingPolicy();
        }
        if (!Strings.isNullOrEmpty(autoScalingPolicy)) {
            // Check if policy is URI or ID. If ID Convert to URI
            if (!autoScalingPolicy.contains("/")) {
                autoScalingPolicy = "projects/" + conf.getProjectId() + "/regions/" + conf.getRegion() + "/autoscalingPolicies/" + autoScalingPolicy;
            }
            builder.setAutoscalingConfig(AutoscalingConfig.newBuilder().setPolicyUri(autoScalingPolicy).build());
        }
        if (conf.getEncryptionKeyName() != null) {
            builder.setEncryptionConfig(EncryptionConfig.newBuilder().setGcePdKmsKeyName(conf.getEncryptionKeyName()).build());
        }
        if (conf.getGcsBucket() != null) {
            builder.setConfigBucket(conf.getGcsBucket());
        }
        Cluster cluster = com.google.cloud.dataproc.v1.Cluster.newBuilder().setClusterName(name).putAllLabels(labels).setConfig(builder.build()).build();
        OperationFuture<Cluster, ClusterOperationMetadata> operationFuture = client.createClusterAsync(conf.getProjectId(), conf.getRegion(), cluster);
        return operationFuture.getMetadata().get();
    } catch (ExecutionException e) {
        cleanUpClusterAfterCreationFailure(name);
        Throwable cause = e.getCause();
        if (cause instanceof ApiException) {
            throw handleApiException((ApiException) cause);
        }
        throw new DataprocRuntimeException(cause);
    }
}

Example 4 with ClusterOperationMetadata

use of com.google.cloud.dataproc.v1.ClusterOperationMetadata in project cdap by caskdata.

the class DataprocTool method main.

public static void main(String[] args) throws Exception {
    Options options = new Options().addOption(new Option("h", "help", false, "Print this usage message.")).addOption(new Option("k", "serviceAccountKey", true, "Google cloud service account key (json format).")).addOption(new Option("p", "project", true, "Google cloud project id.")).addOption(new Option("c", "configFile", true, "File all provisioner settings as a json object.")).addOption(new Option("i", "imageVersion", true, "The image version for the cluster. Defaults to 1.2.")).addOption(new Option("n", "name", true, "Name of the cluster."));
    CommandLineParser parser = new BasicParser();
    CommandLine commandLine = parser.parse(options, args);
    String[] commandArgs = commandLine.getArgs();
    String command = commandArgs.length > 0 ? commandArgs[0] : null;
    // if help is an option, or if there isn't a single 'upgrade' command, print usage and exit.
    if (commandLine.hasOption("h") || commandArgs.length != 1 || !COMMANDS.contains(command.toLowerCase())) {
        printUsage(options);
        System.exit(0);
    }
    if (!commandLine.hasOption('n')) {
        System.err.println("Cluster name must be specified.");
        printUsage(options);
        System.exit(-1);
    }
    DataprocConf conf;
    if (commandLine.hasOption('c')) {
        String configFilePath = commandLine.getOptionValue('c');
        File configFile = new File(configFilePath);
        if (!configFile.isFile()) {
            System.err.println(configFilePath + " is not a file.");
            System.exit(-1);
        }
        try (Reader reader = new FileReader(configFile)) {
            Map<String, String> map = GSON.fromJson(reader, new TypeToken<Map<String, String>>() {
            }.getType());
            conf = DataprocConf.create(map);
        }
    } else {
        if (!commandLine.hasOption('k')) {
            System.err.println("Must specify a service account key or a config file.");
            printUsage(options);
            System.exit(-1);
        }
        if (!commandLine.hasOption('p')) {
            System.err.println("Must specify a project id or a config file.");
            printUsage(options);
            System.exit(-1);
        }
        Map<String, String> properties = new HashMap<>();
        properties.put("accountKey", commandLine.getOptionValue('k'));
        properties.put("projectId", commandLine.getOptionValue('p'));
        conf = DataprocConf.create(properties);
    }
    String imageVersion = commandLine.hasOption('i') ? commandLine.getOptionValue('i') : "1.2";
    String name = commandLine.getOptionValue('n');
    try (DataprocClient client = DataprocClient.fromConf(conf)) {
        if (PROVISION.equalsIgnoreCase(command)) {
            ClusterOperationMetadata createOp = client.createCluster(name, imageVersion, Collections.emptyMap(), false);
            System.out.println(GSON.toJson(createOp));
        } else if (DETAILS.equalsIgnoreCase(command)) {
            Optional<Cluster> cluster = client.getCluster(name);
            if (cluster.isPresent()) {
                System.out.println(GSON.toJson(cluster));
            }
        } else if (DEPROVISION.equalsIgnoreCase(command)) {
            Optional<ClusterOperationMetadata> deleteOp = client.deleteCluster(name);
            if (deleteOp.isPresent()) {
                System.out.println(GSON.toJson(deleteOp));
            }
        } else if (STATUS.equalsIgnoreCase(command)) {
            System.out.println(client.getClusterStatus(name));
        }
    }
}