Examples with Node io.cdap.cdap.runtime.spi.provisioner.Node used on opensource projects

Example 1 with Node

use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.

the class DefaultRuntimeJobTest method testInjector.

@Test
public void testInjector() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().toString());
    LocationFactory locationFactory = new LocalLocationFactory(TEMP_FOLDER.newFile());
    DefaultRuntimeJob defaultRuntimeJob = new DefaultRuntimeJob();
    Arguments systemArgs = new BasicArguments(Collections.singletonMap(SystemArguments.PROFILE_NAME, "test"));
    Node node = new Node("test", Node.Type.MASTER, "127.0.0.1", System.currentTimeMillis(), Collections.emptyMap());
    Cluster cluster = new Cluster("test", ClusterStatus.RUNNING, Collections.singleton(node), Collections.emptyMap());
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(RunIds.generate());
    SimpleProgramOptions programOpts = new SimpleProgramOptions(programRunId.getParent(), systemArgs, new BasicArguments());
    Injector injector = Guice.createInjector(defaultRuntimeJob.createModules(new RuntimeJobEnvironment() {

        @Override
        public LocationFactory getLocationFactory() {
            return locationFactory;
        }

        @Override
        public TwillRunner getTwillRunner() {
            return new NoopTwillRunnerService();
        }

        @Override
        public Map<String, String> getProperties() {
            return Collections.emptyMap();
        }
    }, cConf, programRunId, programOpts));
    injector.getInstance(LogAppenderInitializer.class);
    defaultRuntimeJob.createCoreServices(injector, systemArgs, cluster);
}

Example 2 with Node

use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.

the class RemoteHadoopProvisioner method getMasterExternalIp.

private String getMasterExternalIp(Cluster cluster) {
    Node masterNode = cluster.getNodes().stream().filter(node -> Node.Type.MASTER == node.getType()).findFirst().orElseThrow(() -> new IllegalArgumentException("Cluster has no node of master type: " + cluster));
    String ip = masterNode.getIpAddress();
    if (ip == null) {
        throw new IllegalArgumentException(String.format("External IP is not defined for node '%s' in cluster %s", masterNode.getId(), cluster));
    }
    return ip;
}

Example 3 with Node

use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.

the class DataprocClient method getNode.

private Node getNode(Compute compute, Node.Type type, String zone, String nodeName) throws IOException {
    Instance instance;
    try {
        instance = compute.instances().get(conf.getProjectId(), zone, nodeName).execute();
    } catch (GoogleJsonResponseException e) {
        // this can happen right after a cluster is created
        if (e.getStatusCode() == 404) {
            return new Node(nodeName, Node.Type.UNKNOWN, "", -1L, Collections.emptyMap());
        }
        throw e;
    }
    Map<String, String> properties = new HashMap<>();
    // Dataproc cluster node should only have exactly one network
    instance.getNetworkInterfaces().stream().findFirst().ifPresent(networkInterface -> {
        // if the cluster does not have an external ip then then access config is null
        if (networkInterface.getAccessConfigs() != null) {
            for (AccessConfig accessConfig : networkInterface.getAccessConfigs()) {
                if (accessConfig.getNatIP() != null) {
                    properties.put("ip.external", accessConfig.getNatIP());
                    break;
                }
            }
        }
        properties.put("ip.internal", networkInterface.getNetworkIP());
    });
    long ts;
    try {
        // something like 2018-04-16T12:09:03.943-07:00
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss.SSSX");
        ts = sdf.parse(instance.getCreationTimestamp()).getTime();
    } catch (ParseException | NumberFormatException e) {
        LOG.debug("Fail to parse creation ts {}", instance.getCreationTimestamp(), e);
        ts = -1L;
    }
    // For internal IP only cluster, nodes only have ip.internal.
    String ip = properties.get("ip.external");
    if (ip == null) {
        ip = properties.get("ip.internal");
    }
    return new Node(nodeName, type, ip, ts, properties);
}

Example 4 with Node

use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.

the class EMRClient method getCluster.

/**
 * Get information about the specified cluster. The cluster will not be present if it could not be found.
 *
 * @param id the cluster id
 * @return the cluster information if it exists
 */
public Optional<io.cdap.cdap.runtime.spi.provisioner.Cluster> getCluster(String id) {
    Cluster cluster = describeCluster(id);
    List<Node> nodes = new ArrayList<>();
    nodes.add(new Node("id", Node.Type.MASTER, cluster.getMasterPublicDnsName(), System.currentTimeMillis(), Collections.emptyMap()));
    return Optional.of(new io.cdap.cdap.runtime.spi.provisioner.Cluster(cluster.getId(), convertStatus(cluster.getStatus()), nodes, Collections.emptyMap()));
}

Example 5 with Node

use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.

the class ElasticMapReduceProvisioner method getMasterExternalIp.

private String getMasterExternalIp(Cluster cluster) {
    Node masterNode = cluster.getNodes().stream().filter(node -> Node.Type.MASTER == node.getType()).findFirst().orElseThrow(() -> new IllegalArgumentException("Cluster has no node of master type: " + cluster));
    String ip = masterNode.getIpAddress();
    if (ip == null) {
        throw new IllegalArgumentException(String.format("External IP is not defined for node '%s' in cluster %s", masterNode.getId(), cluster));
    }
    return ip;
}