Examples with DataprocUtils io.cdap.cdap.runtime.spi.common.DataprocUtils used on opensource projects

Example 1 with DataprocUtils

use of io.cdap.cdap.runtime.spi.common.DataprocUtils in project cdap by caskdata.

the class DataprocClient method getFirewallTargetTags.

/**
 * Finds ingress firewall rules for the configured network that matches the required firewall port as
 * defined in {@link FirewallPort}.
 *
 * @return a {@link Collection} of tags that need to be added to the VM to have those firewall rules applies
 * @throws IOException If failed to discover those firewall rules
 */
private List<String> getFirewallTargetTags(Network network, boolean useInternalIP) throws IOException, RetryableProvisionException {
    FirewallList firewalls;
    try {
        firewalls = compute.firewalls().list(conf.getNetworkHostProjectID()).execute();
    } catch (Exception e) {
        handleRetryableExceptions(e);
        throw e;
    }
    List<String> tags = new ArrayList<>();
    Set<FirewallPort> requiredPorts = EnumSet.allOf(FirewallPort.class);
    // Iterate all firewall rules and see if it has ingress rules for all required firewall port.
    for (Firewall firewall : Optional.ofNullable(firewalls.getItems()).orElse(Collections.emptyList())) {
        // network is a url like https://www.googleapis.com/compute/v1/projects/<project>/<region>/networks/<name>
        // we want to get the last section of the path and compare to the configured network name
        int idx = firewall.getNetwork().lastIndexOf('/');
        String networkName = idx >= 0 ? firewall.getNetwork().substring(idx + 1) : firewall.getNetwork();
        if (!networkName.equals(network.getName())) {
            continue;
        }
        String direction = firewall.getDirection();
        if (!"INGRESS".equals(direction) || firewall.getAllowed() == null) {
            continue;
        }
        if (useInternalIP) {
            // private IP blocks in order to be able to communicate with Dataproc.
            try {
                List<IPRange> sourceRanges = Optional.ofNullable(firewall.getSourceRanges()).map(DataprocUtils::parseIPRanges).orElse(Collections.emptyList());
                if (!sourceRanges.isEmpty()) {
                    boolean isPrivate = PRIVATE_IP_RANGES.stream().anyMatch(privateRange -> sourceRanges.stream().anyMatch(privateRange::isOverlap));
                    if (!isPrivate) {
                        continue;
                    }
                }
            } catch (Exception e) {
                LOG.warn("Failed to parse source ranges from firewall rule {}", firewall.getName(), e);
            }
        }
        for (Firewall.Allowed allowed : firewall.getAllowed()) {
            String protocol = allowed.getIPProtocol();
            boolean addTag = false;
            if ("all".equalsIgnoreCase(protocol)) {
                requiredPorts.clear();
                addTag = true;
            } else if ("tcp".equalsIgnoreCase(protocol) && isPortAllowed(allowed.getPorts(), FirewallPort.SSH.port)) {
                requiredPorts.remove(FirewallPort.SSH);
                addTag = true;
            }
            if (addTag && firewall.getTargetTags() != null && !firewall.getTargetTags().isEmpty()) {
                tags.add(firewall.getTargetTags().iterator().next());
            }
        }
    }
    if (!requiredPorts.isEmpty()) {
        String portList = requiredPorts.stream().map(p -> String.valueOf(p.port)).collect(Collectors.joining(","));
        throw new IllegalArgumentException(String.format("Could not find an ingress firewall rule for network '%s' in project '%s' for ports '%s'. " + "Please create a rule to allow incoming traffic on those ports for your IP range.", network.getName(), conf.getNetworkHostProjectID(), portList));
    }
    return tags;
}