Examples with SSHPublicKey io.cdap.cdap.runtime.spi.ssh.SSHPublicKey used on opensource projects

Example 1 with SSHPublicKey

use of io.cdap.cdap.runtime.spi.ssh.SSHPublicKey in project cdap by caskdata.

the class DataprocClient method createCluster.

/**
 * Create a cluster. This will return after the initial request to create the cluster is completed.
 * At this point, the cluster is likely not yet running, but in a provisioning state.
 *
 * @param name         the name of the cluster to create
 * @param imageVersion the image version for the cluster
 * @param labels       labels to set on the cluster
 * @param privateInstance {@code true} to indicate using private instance
 * @return create operation metadata
 * @throws InterruptedException        if the thread was interrupted while waiting for the initial request to complete
 * @throws AlreadyExistsException      if the cluster already exists
 * @throws IOException                 if there was an I/O error talking to Google Compute APIs
 * @throws RetryableProvisionException if there was a non 4xx error code returned
 */
ClusterOperationMetadata createCluster(String name, String imageVersion, Map<String, String> labels, boolean privateInstance) throws RetryableProvisionException, InterruptedException, IOException {
    if (network == null) {
        // yet being used to create cluster.
        throw new IllegalArgumentException("Missing network information");
    }
    try {
        Map<String, String> metadata = new HashMap<>();
        SSHPublicKey publicKey = conf.getPublicKey();
        if (publicKey != null) {
            // Don't fail if there is no public key. It is for tooling case that the key might be generated differently.
            metadata.put("ssh-keys", publicKey.getUser() + ":" + publicKey.getKey());
            // override any os-login that may be set on the project-level metadata
            // this metadata is only needed if ssh is being used to launch the jobs - CDAP-15369
            metadata.put("enable-oslogin", "false");
        }
        // Check if ClusterMetaData is provided and add them.
        metadata.putAll(conf.getClusterMetaData());
        GceClusterConfig.Builder clusterConfig = GceClusterConfig.newBuilder().addServiceAccountScopes(DataprocConf.CLOUD_PLATFORM_SCOPE).setShieldedInstanceConfig(ShieldedInstanceConfig.newBuilder().setEnableSecureBoot(conf.isSecureBootEnabled()).setEnableVtpm(conf.isvTpmEnabled()).setEnableIntegrityMonitoring(conf.isIntegrityMonitoringEnabled()).build()).putAllMetadata(metadata);
        if (conf.getServiceAccount() != null) {
            clusterConfig.setServiceAccount(conf.getServiceAccount());
        }
        if (conf.getZone() != null) {
            clusterConfig.setZoneUri(conf.getZone());
        }
        // subnets are unique within a location, not within a network, which is why these configs are mutually exclusive.
        if (conf.getSubnet() != null) {
            clusterConfig.setSubnetworkUri(conf.getSubnet());
        } else {
            clusterConfig.setNetworkUri(network.getSelfLink());
        }
        // Add any defined Network Tags
        clusterConfig.addAllTags(conf.getNetworkTags());
        boolean internalIPOnly = isInternalIPOnly(network, privateInstance, publicKey != null);
        // if public key is not null that means ssh is used to launch / monitor job on dataproc
        if (publicKey != null) {
            int maxTags = Math.max(0, DataprocConf.MAX_NETWORK_TAGS - clusterConfig.getTagsCount());
            List<String> tags = getFirewallTargetTags(network, internalIPOnly);
            if (tags.size() > maxTags) {
                LOG.warn("No more than 64 tags can be added. Firewall tags ignored: {}", tags.subList(maxTags, tags.size()));
            }
            tags.stream().limit(maxTags).forEach(clusterConfig::addTags);
        }
        // if internal ip is preferred then create dataproc cluster without external ip for better security
        clusterConfig.setInternalIpOnly(internalIPOnly);
        Map<String, String> clusterProperties = new HashMap<>(conf.getClusterProperties());
        // Enable/Disable stackdriver
        clusterProperties.put("dataproc:dataproc.logging.stackdriver.enable", Boolean.toString(conf.isStackdriverLoggingEnabled()));
        clusterProperties.put("dataproc:dataproc.monitoring.stackdriver.enable", Boolean.toString(conf.isStackdriverMonitoringEnabled()));
        DiskConfig workerDiskConfig = DiskConfig.newBuilder().setBootDiskSizeGb(conf.getWorkerDiskGB()).setBootDiskType(conf.getWorkerDiskType()).setNumLocalSsds(0).build();
        InstanceGroupConfig.Builder primaryWorkerConfig = InstanceGroupConfig.newBuilder().setNumInstances(conf.getWorkerNumNodes()).setMachineTypeUri(conf.getWorkerMachineType()).setDiskConfig(workerDiskConfig);
        InstanceGroupConfig.Builder secondaryWorkerConfig = InstanceGroupConfig.newBuilder().setNumInstances(conf.getSecondaryWorkerNumNodes()).setMachineTypeUri(conf.getWorkerMachineType()).setPreemptibility(InstanceGroupConfig.Preemptibility.NON_PREEMPTIBLE).setDiskConfig(workerDiskConfig);
        // Set default concurrency settings for fixed cluster
        if (Strings.isNullOrEmpty(conf.getAutoScalingPolicy()) && !conf.isPredefinedAutoScaleEnabled()) {
            // Set spark.default.parallelism according to cluster size.
            // Spark defaults it to number of current executors, but when we configure the job
            // executors may not have started yet, so this value gets artificially low.
            int defaultConcurrency = Math.max(conf.getTotalWorkerCPUs(), MIN_DEFAULT_CONCURRENCY);
            // Set spark.sql.adaptive.coalescePartitions.initialPartitionNum as 32x of default parallelism,
            // but no more than 8192. This value is used only in spark 3 with adaptive execution and
            // according to our tests spark can handle really large numbers and 32x is a reasonable default.
            int initialPartitionNum = Math.min(Math.max(conf.getTotalWorkerCPUs() * PARTITION_NUM_FACTOR, MIN_INITIAL_PARTITIONS_DEFAULT), MAX_INITIAL_PARTITIONS_DEFAULT);
            clusterProperties.putIfAbsent("spark:spark.default.parallelism", Integer.toString(defaultConcurrency));
            clusterProperties.putIfAbsent("spark:spark.sql.adaptive.coalescePartitions.initialPartitionNum", Integer.toString(initialPartitionNum));
        }
        SoftwareConfig.Builder softwareConfigBuilder = SoftwareConfig.newBuilder().putAllProperties(clusterProperties);
        // Use image version only if custom Image URI is not specified, otherwise may cause image version conflicts
        if (conf.getCustomImageUri() == null || conf.getCustomImageUri().isEmpty()) {
            softwareConfigBuilder.setImageVersion(imageVersion);
        } else {
            // If custom Image URI is specified, use that for cluster creation
            primaryWorkerConfig.setImageUri(conf.getCustomImageUri());
            secondaryWorkerConfig.setImageUri(conf.getCustomImageUri());
        }
        ClusterConfig.Builder builder = ClusterConfig.newBuilder().setEndpointConfig(EndpointConfig.newBuilder().setEnableHttpPortAccess(conf.isComponentGatewayEnabled()).build()).setMasterConfig(InstanceGroupConfig.newBuilder().setNumInstances(conf.getMasterNumNodes()).setMachineTypeUri(conf.getMasterMachineType()).setDiskConfig(DiskConfig.newBuilder().setBootDiskType(conf.getMasterDiskType()).setBootDiskSizeGb(conf.getMasterDiskGB()).setNumLocalSsds(0).build()).build()).setWorkerConfig(primaryWorkerConfig.build()).setSecondaryWorkerConfig(secondaryWorkerConfig.build()).setGceClusterConfig(clusterConfig.build()).setSoftwareConfig(softwareConfigBuilder);
        // Cluster TTL if one should be set
        if (conf.getIdleTTLMinutes() > 0) {
            long seconds = TimeUnit.MINUTES.toSeconds(conf.getIdleTTLMinutes());
            builder.setLifecycleConfig(LifecycleConfig.newBuilder().setIdleDeleteTtl(Duration.newBuilder().setSeconds(seconds).build()).build());
        }
        // Add any Node Initialization action scripts
        for (String action : conf.getInitActions()) {
            builder.addInitializationActions(NodeInitializationAction.newBuilder().setExecutableFile(action).build());
        }
        // Set Auto Scaling Policy
        String autoScalingPolicy = conf.getAutoScalingPolicy();
        if (conf.isPredefinedAutoScaleEnabled()) {
            PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(conf);
            autoScalingPolicy = predefinedAutoScaling.createPredefinedAutoScalingPolicy();
        }
        if (!Strings.isNullOrEmpty(autoScalingPolicy)) {
            // Check if policy is URI or ID. If ID Convert to URI
            if (!autoScalingPolicy.contains("/")) {
                autoScalingPolicy = "projects/" + conf.getProjectId() + "/regions/" + conf.getRegion() + "/autoscalingPolicies/" + autoScalingPolicy;
            }
            builder.setAutoscalingConfig(AutoscalingConfig.newBuilder().setPolicyUri(autoScalingPolicy).build());
        }
        if (conf.getEncryptionKeyName() != null) {
            builder.setEncryptionConfig(EncryptionConfig.newBuilder().setGcePdKmsKeyName(conf.getEncryptionKeyName()).build());
        }
        if (conf.getGcsBucket() != null) {
            builder.setConfigBucket(conf.getGcsBucket());
        }
        Cluster cluster = com.google.cloud.dataproc.v1.Cluster.newBuilder().setClusterName(name).putAllLabels(labels).setConfig(builder.build()).build();
        OperationFuture<Cluster, ClusterOperationMetadata> operationFuture = client.createClusterAsync(conf.getProjectId(), conf.getRegion(), cluster);
        return operationFuture.getMetadata().get();
    } catch (ExecutionException e) {
        cleanUpClusterAfterCreationFailure(name);
        Throwable cause = e.getCause();
        if (cause instanceof ApiException) {
            throw handleApiException((ApiException) cause);
        }
        throw new DataprocRuntimeException(cause);
    }
}

Example 2 with SSHPublicKey

use of io.cdap.cdap.runtime.spi.ssh.SSHPublicKey in project cdap by caskdata.

the class ExistingDataprocProvisioner method createCluster.

@Override
public Cluster createCluster(ProvisionerContext context) throws Exception {
    Map<String, String> contextProperties = createContextProperties(context);
    DataprocConf conf = DataprocConf.create(contextProperties);
    if (context.getRuntimeMonitorType() == RuntimeMonitorType.SSH) {
        String sshUser = contextProperties.get(SSH_USER);
        String sshKey = contextProperties.get(SSH_KEY);
        if (Strings.isNullOrEmpty(sshUser) || Strings.isNullOrEmpty(sshKey)) {
            throw new DataprocRuntimeException("SSH User and key are required for monitoring through SSH.");
        }
        SSHKeyPair sshKeyPair = new SSHKeyPair(new SSHPublicKey(sshUser, ""), () -> sshKey.getBytes(StandardCharsets.UTF_8));
        // The ssh context shouldn't be null, but protect it in case there is platform bug
        Optional.ofNullable(context.getSSHContext()).ifPresent(c -> c.setSSHKeyPair(sshKeyPair));
    }
    String clusterName = contextProperties.get(CLUSTER_NAME);
    try (DataprocClient client = DataprocClient.fromConf(conf, false)) {
        try {
            client.updateClusterLabels(clusterName, getSystemLabels());
        } catch (DataprocRuntimeException e) {
            // Only log the stacktrace if trace log level is enabled
            if (LOG.isTraceEnabled()) {
                LOG.trace("Cannot update cluster labels due to {}", e.getMessage(), e);
            } else {
                LOG.debug("Cannot update cluster labels due to {}", e.getMessage());
            }
        }
        return client.getCluster(clusterName).filter(c -> c.getStatus() == ClusterStatus.RUNNING).orElseThrow(() -> new DataprocRuntimeException("Dataproc cluster " + clusterName + " does not exist or not in running state."));
    }
}

Example 3 with SSHPublicKey

use of io.cdap.cdap.runtime.spi.ssh.SSHPublicKey in project cdap by caskdata.

the class DataprocConf method create.

/**
 * Create the conf from a property map while also performing validation.
 *
 * @param publicKey an optional {@link SSHPublicKey} for the configuration
 * @throws IllegalArgumentException if it is an invalid config
 */
static DataprocConf create(Map<String, String> properties, @Nullable SSHPublicKey publicKey) {
    String accountKey = getString(properties, "accountKey");
    if (accountKey == null || AUTO_DETECT.equals(accountKey)) {
        String endPoint = getString(properties, TOKEN_ENDPOINT_KEY);
        try {
            ComputeEngineCredentials.getOrCreate(endPoint);
        } catch (IOException e) {
            throw new IllegalArgumentException("Unable to get credentials from the environment. " + "Please explicitly set the account key.", e);
        }
    }
    String projectId = getString(properties, PROJECT_ID_KEY);
    if (projectId == null || AUTO_DETECT.equals(projectId)) {
        projectId = DataprocUtils.getSystemProjectId();
    }
    String zone = getString(properties, "zone");
    String region = getString(properties, "region");
    if (region == null || AUTO_DETECT.equals(region)) {
        // If it does, derived region from the provided zone; otherwise, use the system zone.
        if (zone == null || AUTO_DETECT.equals(zone)) {
            region = DataprocUtils.getRegionFromZone(DataprocUtils.getSystemZone());
        } else {
            region = DataprocUtils.getRegionFromZone(zone);
        }
    }
    if (zone == null || AUTO_DETECT.equals(zone)) {
        // Region is always set so that zone can be omitted
        zone = null;
    } else {
        // Make sure the zone provided match with the region
        if (!zone.startsWith(region + "-")) {
            throw new IllegalArgumentException("Provided zone " + zone + " is not in the region " + region);
        }
    }
    String networkHostProjectID = getString(properties, NETWORK_HOST_PROJECT_ID);
    String network = getString(properties, NETWORK);
    if (network == null || AUTO_DETECT.equals(network)) {
        network = null;
    }
    String subnet = getString(properties, "subnet");
    int masterNumNodes = getInt(properties, "masterNumNodes", 1);
    if (masterNumNodes != 1 && masterNumNodes != 3) {
        throw new IllegalArgumentException(String.format("Invalid config 'masterNumNodes' = %d. Master nodes must be either 1 or 3.", masterNumNodes));
    }
    int workerNumNodes = getInt(properties, WORKER_NUM_NODES, 2);
    int secondaryWorkerNumNodes = getInt(properties, SECONDARY_WORKER_NUM_NODES, 0);
    String autoScalingPolicy = getString(properties, AUTOSCALING_POLICY);
    boolean enablePredefinedAutoScaling = Boolean.parseBoolean(properties.getOrDefault(PREDEFINED_AUTOSCALE_ENABLED, "false"));
    if (enablePredefinedAutoScaling) {
        workerNumNodes = PredefinedAutoScaling.getPrimaryWorkerInstances();
        secondaryWorkerNumNodes = PredefinedAutoScaling.getMinSecondaryWorkerInstances();
        // The policy will be created while cluster provisioning
        autoScalingPolicy = "";
    }
    if (workerNumNodes == 1) {
        throw new IllegalArgumentException("Invalid config 'workerNumNodes' = 1. Worker nodes must either be zero for a single node cluster, " + "or at least 2 for a multi node cluster.");
    }
    if (secondaryWorkerNumNodes < 0) {
        throw new IllegalArgumentException(String.format("Invalid config 'secondaryWorkerNumNodes' = %d. The value must be 0 or greater.", secondaryWorkerNumNodes));
    }
    // TODO: more extensive validation. Each cpu number has a different allowed memory range
    // for example, 1 cpu requires memory from 3.5gb to 6.5gb in .25gb increments
    // 3 cpu requires memory from 3.6gb to 26gb in .25gb increments
    int masterCPUs = getInt(properties, "masterCPUs", 4);
    int workerCPUs = getInt(properties, "workerCPUs", 4);
    int masterMemoryGB = getInt(properties, "masterMemoryMB", 15 * 1024);
    int workerMemoryGB = getInt(properties, "workerMemoryMB", 15 * 1024);
    int masterDiskGB = getInt(properties, "masterDiskGB", 1000);
    String masterDiskType = getString(properties, "masterDiskType");
    String masterMachineType = getString(properties, "masterMachineType");
    if (masterDiskType == null) {
        masterDiskType = "pd-standard";
    }
    int workerDiskGB = getInt(properties, "workerDiskGB", 1000);
    String workerDiskType = getString(properties, "workerDiskType");
    String workerMachineType = getString(properties, "workerMachineType");
    if (workerDiskType == null) {
        workerDiskType = "pd-standard";
    }
    long pollCreateDelay = getLong(properties, "pollCreateDelay", 60);
    long pollCreateJitter = getLong(properties, "pollCreateJitter", 20);
    long pollDeleteDelay = getLong(properties, "pollDeleteDelay", 30);
    long pollInterval = getLong(properties, "pollInterval", 2);
    String serviceAccount = getString(properties, "serviceAccount");
    boolean preferExternalIP = Boolean.parseBoolean(properties.get(PREFER_EXTERNAL_IP));
    // By default stackdriver is enabled. This is for backward compatibility
    boolean stackdriverLoggingEnabled = Boolean.parseBoolean(properties.getOrDefault(STACKDRIVER_LOGGING_ENABLED, "true"));
    boolean stackdriverMonitoringEnabled = Boolean.parseBoolean(properties.getOrDefault(STACKDRIVER_MONITORING_ENABLED, "true"));
    boolean componentGatewayEnabled = Boolean.parseBoolean(properties.get(COMPONENT_GATEWAY_ENABLED));
    boolean skipDelete = Boolean.parseBoolean(properties.get(SKIP_DELETE));
    Map<String, String> clusterPropOverrides = DataprocUtils.parseKeyValueConfig(getString(properties, "clusterProperties"), ";", "=");
    Map<String, String> clusterProps = properties.entrySet().stream().filter(e -> CLUSTER_PROPERTIES_PATTERN.matcher(e.getKey()).find()).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    clusterProps.putAll(clusterPropOverrides);
    String imageVersion = getString(properties, IMAGE_VERSION);
    String customImageUri = getString(properties, CUSTOM_IMAGE_URI);
    String gcpCmekKeyName = getString(properties, ENCRYPTION_KEY_NAME);
    String gcpCmekBucket = getString(properties, "gcsBucket");
    Map<String, String> clusterMetaData = Collections.unmodifiableMap(DataprocUtils.parseKeyValueConfig(getString(properties, CLUSTER_META_DATA), ";", "\\|"));
    Map<String, String> clusterLabels = Collections.unmodifiableMap(DataprocUtils.parseKeyValueConfig(getString(properties, CLUSTER_LABELS), ";", "\\|"));
    String networkTagsProperty = Optional.ofNullable(getString(properties, "networkTags")).orElse("");
    List<String> networkTags = Collections.unmodifiableList(Arrays.stream(networkTagsProperty.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()));
    if (networkTags.size() > MAX_NETWORK_TAGS) {
        throw new IllegalArgumentException("Number of network tags cannot be more than " + MAX_NETWORK_TAGS);
    }
    String initActions = getString(properties, "initActions");
    boolean runtimeJobManagerEnabled = Boolean.parseBoolean(properties.get(RUNTIME_JOB_MANAGER));
    int idleTTL = getInt(properties, CLUSTER_IDLE_TTL_MINUTES, CLUSTER_IDLE_TTL_MINUTES_DEFAULT);
    String tokenEndpoint = getString(properties, TOKEN_ENDPOINT_KEY);
    boolean secureBootEnabled = Boolean.parseBoolean(properties.getOrDefault(SECURE_BOOT_ENABLED, "false"));
    boolean vTpmEnabled = Boolean.parseBoolean(properties.getOrDefault(VTPM_ENABLED, "false"));
    boolean integrityMonitoringEnabled = Boolean.parseBoolean(properties.getOrDefault(INTEGRITY_MONITORING_ENABLED, "false"));
    boolean clusterReuseEnabled = Boolean.parseBoolean(properties.getOrDefault(CLUSTER_REUSE_ENABLED, "true"));
    int clusterReuseThresholdMinutes = getInt(properties, CLUSTER_REUSE_THRESHOLD_MINUTES, CLUSTER_REUSE_THRESHOLD_MINUTES_DEFAULT);
    String clusterReuseKey = null;
    if (clusterReuseEnabled) {
        try {
            MessageDigest digest = MessageDigest.getInstance("SHA-1");
            digest.update(properties.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(",")).getBytes(StandardCharsets.UTF_8));
            clusterReuseKey = String.format("%040x", new BigInteger(1, digest.digest()));
        } catch (NoSuchAlgorithmException e) {
            throw new IllegalStateException("SHA-1 algorithm is not available for cluster reuse", e);
        }
    }
    int computeReadTimeout = getInt(properties, COMPUTE_HTTP_REQUEST_READ_TIMEOUT, COMPUTE_HTTP_REQUEST_READ_TIMEOUT_DEFAULT);
    int computeConnectionTimeout = getInt(properties, COMPUTE_HTTP_REQUEST_CONNECTION_TIMEOUT, COMPUTE_HTTP_REQUEST_CONNECTION_TIMEOUT_DEFAULT);
    String rootUrl = getString(properties, ROOT_URL);
    return new DataprocConf(accountKey, region, zone, projectId, networkHostProjectID, network, subnet, masterNumNodes, masterCPUs, masterMemoryGB, masterDiskGB, masterDiskType, masterMachineType, workerNumNodes, secondaryWorkerNumNodes, workerCPUs, workerMemoryGB, workerDiskGB, workerDiskType, workerMachineType, pollCreateDelay, pollCreateJitter, pollDeleteDelay, pollInterval, gcpCmekKeyName, gcpCmekBucket, serviceAccount, preferExternalIP, stackdriverLoggingEnabled, stackdriverMonitoringEnabled, componentGatewayEnabled, skipDelete, publicKey, imageVersion, customImageUri, clusterMetaData, clusterLabels, networkTags, initActions, runtimeJobManagerEnabled, clusterProps, autoScalingPolicy, idleTTL, tokenEndpoint, secureBootEnabled, vTpmEnabled, integrityMonitoringEnabled, clusterReuseEnabled, clusterReuseThresholdMinutes, clusterReuseKey, enablePredefinedAutoScaling, computeReadTimeout, computeConnectionTimeout, rootUrl);
}

Example 4 with SSHPublicKey

use of io.cdap.cdap.runtime.spi.ssh.SSHPublicKey in project cdap by caskdata.

the class RemoteHadoopConf method fromProperties.

/**
 * Create the conf from a property map while also performing validation.
 */
public static RemoteHadoopConf fromProperties(Map<String, String> properties) {
    String host = getString(properties, "host");
    String user = getString(properties, "user");
    String privateKey = getString(properties, "sshKey");
    SSHKeyPair keyPair = new SSHKeyPair(new SSHPublicKey(user, ""), () -> privateKey.getBytes(StandardCharsets.UTF_8));
    return new RemoteHadoopConf(keyPair, host, properties.get("initializationAction"), properties.get("kerberosPrincipal"), properties.get("kerberosKeytabPath"));
}

Example 5 with SSHPublicKey

use of io.cdap.cdap.runtime.spi.ssh.SSHPublicKey in project cdap by caskdata.

the class DefaultSSHContext method generate.

@Override
public SSHKeyPair generate(String user, int bits) throws KeyException {
    JSch jsch = new JSch();
    try {
        KeyPair keyPair = KeyPair.genKeyPair(jsch, KeyPair.RSA, bits);
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        keyPair.writePublicKey(bos, user);
        SSHPublicKey publicKey = new SSHPublicKey(user, new String(bos.toByteArray(), StandardCharsets.UTF_8));
        bos.reset();
        keyPair.writePrivateKey(bos);
        byte[] privateKey = bos.toByteArray();
        return new SSHKeyPair(publicKey, () -> privateKey);
    } catch (JSchException e) {
        throw new KeyException("Failed to generate ssh key pair", e);
    }
}