use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.
the class DefaultRuntimeJobTest method testInjector.
@Test
public void testInjector() throws Exception {
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().toString());
LocationFactory locationFactory = new LocalLocationFactory(TEMP_FOLDER.newFile());
DefaultRuntimeJob defaultRuntimeJob = new DefaultRuntimeJob();
Arguments systemArgs = new BasicArguments(Collections.singletonMap(SystemArguments.PROFILE_NAME, "test"));
Node node = new Node("test", Node.Type.MASTER, "127.0.0.1", System.currentTimeMillis(), Collections.emptyMap());
Cluster cluster = new Cluster("test", ClusterStatus.RUNNING, Collections.singleton(node), Collections.emptyMap());
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(RunIds.generate());
SimpleProgramOptions programOpts = new SimpleProgramOptions(programRunId.getParent(), systemArgs, new BasicArguments());
Injector injector = Guice.createInjector(defaultRuntimeJob.createModules(new RuntimeJobEnvironment() {
@Override
public LocationFactory getLocationFactory() {
return locationFactory;
}
@Override
public TwillRunner getTwillRunner() {
return new NoopTwillRunnerService();
}
@Override
public Map<String, String> getProperties() {
return Collections.emptyMap();
}
}, cConf, programRunId, programOpts));
injector.getInstance(LogAppenderInitializer.class);
defaultRuntimeJob.createCoreServices(injector, systemArgs, cluster);
}
use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.
the class RemoteHadoopProvisioner method getMasterExternalIp.
private String getMasterExternalIp(Cluster cluster) {
Node masterNode = cluster.getNodes().stream().filter(node -> Node.Type.MASTER == node.getType()).findFirst().orElseThrow(() -> new IllegalArgumentException("Cluster has no node of master type: " + cluster));
String ip = masterNode.getIpAddress();
if (ip == null) {
throw new IllegalArgumentException(String.format("External IP is not defined for node '%s' in cluster %s", masterNode.getId(), cluster));
}
return ip;
}
use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.
the class DataprocClient method getNode.
private Node getNode(Compute compute, Node.Type type, String zone, String nodeName) throws IOException {
Instance instance;
try {
instance = compute.instances().get(conf.getProjectId(), zone, nodeName).execute();
} catch (GoogleJsonResponseException e) {
// this can happen right after a cluster is created
if (e.getStatusCode() == 404) {
return new Node(nodeName, Node.Type.UNKNOWN, "", -1L, Collections.emptyMap());
}
throw e;
}
Map<String, String> properties = new HashMap<>();
// Dataproc cluster node should only have exactly one network
instance.getNetworkInterfaces().stream().findFirst().ifPresent(networkInterface -> {
// if the cluster does not have an external ip then then access config is null
if (networkInterface.getAccessConfigs() != null) {
for (AccessConfig accessConfig : networkInterface.getAccessConfigs()) {
if (accessConfig.getNatIP() != null) {
properties.put("ip.external", accessConfig.getNatIP());
break;
}
}
}
properties.put("ip.internal", networkInterface.getNetworkIP());
});
long ts;
try {
// something like 2018-04-16T12:09:03.943-07:00
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss.SSSX");
ts = sdf.parse(instance.getCreationTimestamp()).getTime();
} catch (ParseException | NumberFormatException e) {
LOG.debug("Fail to parse creation ts {}", instance.getCreationTimestamp(), e);
ts = -1L;
}
// For internal IP only cluster, nodes only have ip.internal.
String ip = properties.get("ip.external");
if (ip == null) {
ip = properties.get("ip.internal");
}
return new Node(nodeName, type, ip, ts, properties);
}
use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.
the class EMRClient method getCluster.
/**
* Get information about the specified cluster. The cluster will not be present if it could not be found.
*
* @param id the cluster id
* @return the cluster information if it exists
*/
public Optional<io.cdap.cdap.runtime.spi.provisioner.Cluster> getCluster(String id) {
Cluster cluster = describeCluster(id);
List<Node> nodes = new ArrayList<>();
nodes.add(new Node("id", Node.Type.MASTER, cluster.getMasterPublicDnsName(), System.currentTimeMillis(), Collections.emptyMap()));
return Optional.of(new io.cdap.cdap.runtime.spi.provisioner.Cluster(cluster.getId(), convertStatus(cluster.getStatus()), nodes, Collections.emptyMap()));
}
use of io.cdap.cdap.runtime.spi.provisioner.Node in project cdap by caskdata.
the class ElasticMapReduceProvisioner method getMasterExternalIp.
private String getMasterExternalIp(Cluster cluster) {
Node masterNode = cluster.getNodes().stream().filter(node -> Node.Type.MASTER == node.getType()).findFirst().orElseThrow(() -> new IllegalArgumentException("Cluster has no node of master type: " + cluster));
String ip = masterNode.getIpAddress();
if (ip == null) {
throw new IllegalArgumentException(String.format("External IP is not defined for node '%s' in cluster %s", masterNode.getId(), cluster));
}
return ip;
}
Aggregations