use of edu.iu.dsc.tws.rsched.bootstrap.ZKController in project twister2 by DSC-SPIDAL.
the class AuroraWorker method initializeWithZooKeeper.
public void initializeWithZooKeeper() {
long startTime = System.currentTimeMillis();
String workerHostPort = workerAddress.getHostAddress() + ":" + workerPort;
int numberOfWorkers = job.getJobResources().getNoOfContainers();
zkController = new ZKController(config, job.getJobName(), workerHostPort, numberOfWorkers);
zkController.initialize();
long duration = System.currentTimeMillis() - startTime;
System.out.println("Initialization for the worker: " + zkController.getWorkerNetworkInfo() + " took: " + duration + "ms");
}
use of edu.iu.dsc.tws.rsched.bootstrap.ZKController in project twister2 by DSC-SPIDAL.
the class TCPNetwork method initialize.
public void initialize() {
NetworkInfo networkInfo = new NetworkInfo(-1);
// we need to first start the server part of tcp
// lets first start the channel
channel = new TCPChannel(config, networkInfo);
channel.startFirstPhase();
// first lets intialize the zk
int numberOfWorkers = job.getJobResources().getNoOfContainers();
zkController = new ZKController(config, job.getJobName(), workerUniqueId, numberOfWorkers);
zkController.initialize();
// the amount of time to wait for all workers to join a job
int timeLimit = ZKContext.maxWaitTimeForAllWorkersToJoin(config);
List<WorkerNetworkInfo> workerNetworkInfoList = zkController.waitForAllWorkersToJoin(timeLimit);
if (workerNetworkInfoList == null) {
throw new RuntimeException("Error getting the worker list from ZooKeeper");
}
List<NetworkInfo> networkInfos = new ArrayList<>();
NetworkInfo thisNet = null;
for (WorkerNetworkInfo info : workerNetworkInfoList) {
NetworkInfo netInfo = new NetworkInfo(info.getWorkerID());
netInfo.addProperty(TCPContext.NETWORK_HOSTNAME, info.getWorkerIP());
netInfo.addProperty(TCPContext.NETWORK_PORT, info.getWorkerPort());
// todo: we need to have tcp information from worker info
if (workerUniqueId.equals(info.getWorkerName())) {
thisNet = netInfo;
}
networkInfos.add(netInfo);
}
// now intialize with zookeeper
channel.startSecondPhase(networkInfos, thisNet);
TWSTCPChannel twstcpChannel = new TWSTCPChannel(config, taskPlan.getThisExecutor(), channel);
// now lets create the dataflow communication
dataFlowCommunication = new MPIDataFlowCommunication();
dataFlowCommunication.init(config, taskPlan, twstcpChannel);
}
Aggregations