Examples with Cluster org.apache.asterix.event.schema.yarnCluster.Cluster used on opensource projects

Example 1 with Cluster

use of org.apache.asterix.event.schema.yarnCluster.Cluster in project asterixdb by apache.

the class AsterixYARNInstanceUtil method setUp.

public YarnConfiguration setUp() throws Exception {
    File asterixProjectDir = new File(System.getProperty("user.dir"));
    File installerTargetDir = new File(asterixProjectDir, "target");
    String[] dirsInTarget = installerTargetDir.list(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return new File(dir, name).isDirectory() && name.startsWith("asterix-yarn") && name.endsWith("binary-assembly");
        }
    });
    if (dirsInTarget.length != 1) {
        throw new IllegalStateException("Could not find binary to run YARN integration test with");
    }
    aoyaHome = installerTargetDir.getAbsolutePath() + File.separator + dirsInTarget[0];
    File asterixServerInstallerDir = new File(aoyaHome, "asterix");
    String[] zipsInFolder = asterixServerInstallerDir.list(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return name.startsWith("asterix-server") && name.endsWith("binary-assembly.zip");
        }
    });
    if (zipsInFolder.length != 1) {
        throw new IllegalStateException("Could not find server binary to run YARN integration test with");
    }
    aoyaServerPath = asterixServerInstallerDir.getAbsolutePath() + File.separator + zipsInFolder[0];
    configPath = aoyaHome + File.separator + "configs" + File.separator + "local.xml";
    parameterPath = aoyaHome + File.separator + "conf" + File.separator + "base-asterix-configuration.xml";
    YARNCluster.getInstance().setup();
    appConf = new YarnConfiguration();
    File baseDir = new File("./target/hdfs/").getAbsoluteFile();
    FileUtil.fullyDelete(baseDir);
    appConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(appConf);
    MiniDFSCluster hdfsCluster = builder.build();
    miniCluster = YARNCluster.getInstance().getCluster();
    appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort());
    miniCluster.init(appConf);
    Cluster defaultConfig = Utils.parseYarnClusterConfig(configPath);
    for (Node n : defaultConfig.getNode()) {
        n.setClusterIp(MiniYARNCluster.getHostname());
    }
    defaultConfig.getMasterNode().setClusterIp(MiniYARNCluster.getHostname());
    configPath = "target" + File.separator + "localized-aoya-config.xml";
    Utils.writeYarnClusterConfig(configPath, defaultConfig);
    miniCluster.start();
    appConf = new YarnConfiguration(miniCluster.getConfig());
    appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort());
    //TODO:why must I do this!? what is not being passed properly via environment variables???
    appConf.writeXml(new FileOutputStream("target" + File.separator + "yarn-site.xml"));
    //once the cluster is created, you can get its configuration
    //with the binding details to the cluster added from the minicluster
    FileSystem fs = FileSystem.get(appConf);
    Path instanceState = new Path(fs.getHomeDirectory(), AsterixYARNClient.CONF_DIR_REL + INSTANCE_NAME + "/");
    fs.delete(instanceState, true);
    Assert.assertFalse(fs.exists(instanceState));
    File outdir = new File(PATH_ACTUAL);
    outdir.mkdirs();
    return appConf;
}

Example 2 with Cluster

use of org.apache.asterix.event.schema.yarnCluster.Cluster in project asterixdb by apache.

the class Utils method getCCHostname.

public static String getCCHostname(String instanceName, Configuration conf) throws YarnException {
    try {
        FileSystem fs = FileSystem.get(conf);
        String instanceFolder = instanceName + "/";
        String pathSuffix = CONF_DIR_REL + instanceFolder + "cluster-config.xml";
        Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix);
        File tmp = File.createTempFile("cluster-config", "xml");
        tmp.deleteOnExit();
        fs.copyToLocalFile(dstConf, new Path(tmp.getPath()));
        JAXBContext clusterConf = JAXBContext.newInstance(Cluster.class);
        Unmarshaller unm = clusterConf.createUnmarshaller();
        Cluster cl = (Cluster) unm.unmarshal(tmp);
        String ccIp = cl.getMasterNode().getClientIp();
        return ccIp;
    } catch (IOException | JAXBException e) {
        throw new YarnException(e);
    }
}

Example 3 with Cluster

use of org.apache.asterix.event.schema.yarnCluster.Cluster in project asterixdb by apache.

the class Utils method parseYarnClusterConfig.

/**
     * Simply parses out the YARN cluster config and instantiates it into a nice object.
     *
     * @return The object representing the configuration
     * @throws FileNotFoundException
     * @throws JAXBException
     */
public static Cluster parseYarnClusterConfig(String path) throws YarnException {
    try {
        File f = new File(path);
        JAXBContext configCtx = JAXBContext.newInstance(Cluster.class);
        Unmarshaller unmarshaller = configCtx.createUnmarshaller();
        Cluster cl = (Cluster) unmarshaller.unmarshal(f);
        return cl;
    } catch (JAXBException e) {
        throw new YarnException(e);
    }
}