Examples with Node org.apache.asterix.event.schema.yarnCluster.Node used on opensource projects

Example 1 with Node

use of org.apache.asterix.event.schema.yarnCluster.Node in project asterixdb by apache.

the class AsterixYARNInstanceUtil method setUp.

public YarnConfiguration setUp() throws Exception {
    File asterixProjectDir = new File(System.getProperty("user.dir"));
    File installerTargetDir = new File(asterixProjectDir, "target");
    String[] dirsInTarget = installerTargetDir.list(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return new File(dir, name).isDirectory() && name.startsWith("asterix-yarn") && name.endsWith("binary-assembly");
        }
    });
    if (dirsInTarget.length != 1) {
        throw new IllegalStateException("Could not find binary to run YARN integration test with");
    }
    aoyaHome = installerTargetDir.getAbsolutePath() + File.separator + dirsInTarget[0];
    File asterixServerInstallerDir = new File(aoyaHome, "asterix");
    String[] zipsInFolder = asterixServerInstallerDir.list(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return name.startsWith("asterix-server") && name.endsWith("binary-assembly.zip");
        }
    });
    if (zipsInFolder.length != 1) {
        throw new IllegalStateException("Could not find server binary to run YARN integration test with");
    }
    aoyaServerPath = asterixServerInstallerDir.getAbsolutePath() + File.separator + zipsInFolder[0];
    configPath = aoyaHome + File.separator + "configs" + File.separator + "local.xml";
    parameterPath = aoyaHome + File.separator + "conf" + File.separator + "base-asterix-configuration.xml";
    YARNCluster.getInstance().setup();
    appConf = new YarnConfiguration();
    File baseDir = new File("./target/hdfs/").getAbsoluteFile();
    FileUtil.fullyDelete(baseDir);
    appConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(appConf);
    MiniDFSCluster hdfsCluster = builder.build();
    miniCluster = YARNCluster.getInstance().getCluster();
    appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort());
    miniCluster.init(appConf);
    Cluster defaultConfig = Utils.parseYarnClusterConfig(configPath);
    for (Node n : defaultConfig.getNode()) {
        n.setClusterIp(MiniYARNCluster.getHostname());
    }
    defaultConfig.getMasterNode().setClusterIp(MiniYARNCluster.getHostname());
    configPath = "target" + File.separator + "localized-aoya-config.xml";
    Utils.writeYarnClusterConfig(configPath, defaultConfig);
    miniCluster.start();
    appConf = new YarnConfiguration(miniCluster.getConfig());
    appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort());
    //TODO:why must I do this!? what is not being passed properly via environment variables???
    appConf.writeXml(new FileOutputStream("target" + File.separator + "yarn-site.xml"));
    //once the cluster is created, you can get its configuration
    //with the binding details to the cluster added from the minicluster
    FileSystem fs = FileSystem.get(appConf);
    Path instanceState = new Path(fs.getHomeDirectory(), AsterixYARNClient.CONF_DIR_REL + INSTANCE_NAME + "/");
    fs.delete(instanceState, true);
    Assert.assertFalse(fs.exists(instanceState));
    File outdir = new File(PATH_ACTUAL);
    outdir.mkdirs();
    return appConf;
}

Example 2 with Node

use of org.apache.asterix.event.schema.yarnCluster.Node in project asterixdb by apache.

the class AsterixYARNClient method writeAsterixConfig.

/**
     * Retrieves necessary information from the cluster configuration and splices it into the Asterix configuration parameters
     *
     * @param cluster
     * @throws FileNotFoundException
     * @throws IOException
     */
private void writeAsterixConfig(Cluster cluster) throws FileNotFoundException, IOException {
    String metadataNodeId = Utils.getMetadataNode(cluster).getId();
    String asterixInstanceName = instanceName;
    AsterixConfiguration configuration = locateConfig();
    readConfigParams(configuration);
    String version = Utils.getAsterixVersionFromClasspath();
    configuration.setVersion(version);
    configuration.setInstanceName(asterixInstanceName);
    List<Store> stores = new ArrayList<Store>();
    String storeDir = cluster.getStore().trim();
    for (Node node : cluster.getNode()) {
        String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
        String[] nodeIdDevice = iodevices.split(",");
        StringBuilder nodeStores = new StringBuilder();
        for (int i = 0; i < nodeIdDevice.length; i++) {
            nodeStores.append(nodeIdDevice[i] + File.separator + storeDir + ",");
        }
        //remove last comma
        nodeStores.deleteCharAt(nodeStores.length() - 1);
        stores.add(new Store(node.getId(), nodeStores.toString()));
    }
    configuration.setStore(stores);
    List<Coredump> coredump = new ArrayList<Coredump>();
    String coredumpdir = null;
    List<TransactionLogDir> txnLogDirs = new ArrayList<TransactionLogDir>();
    String txnLogDir = null;
    for (Node node : cluster.getNode()) {
        coredumpdir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir();
        coredump.add(new Coredump(node.getId(), coredumpdir + "coredump" + File.separator));
        //node or cluster-wide
        txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir();
        txnLogDirs.add(new TransactionLogDir(node.getId(), txnLogDir + (txnLogDir.charAt(txnLogDir.length() - 1) == File.separatorChar ? File.separator : "") + //if the string doesn't have a trailing / add one
        "txnLogs" + File.separator));
    }
    configuration.setMetadataNode(metadataNodeId);
    configuration.setCoredump(coredump);
    configuration.setTransactionLogDir(txnLogDirs);
    FileOutputStream os = new FileOutputStream(MERGED_PARAMETERS_PATH);
    try {
        JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
        Marshaller marshaller = ctx.createMarshaller();
        marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
        marshaller.marshal(configuration, os);
    } catch (JAXBException e) {
        throw new IOException(e);
    } finally {
        os.close();
    }
}

Example 3 with Node

use of org.apache.asterix.event.schema.yarnCluster.Node in project asterixdb by apache.

the class AsterixApplicationMaster method requestResources.

/**
     * @param c
     *            The cluster exception to attempt to alocate with the RM
     * @throws YarnException
     */
private void requestResources(Cluster c) throws YarnException, UnknownHostException {
    //set memory
    if (c.getCcContainerMem() != null) {
        ccMem = Integer.parseInt(c.getCcContainerMem());
    } else {
        ccMem = CC_MEMORY_MBS_DEFAULT;
    }
    if (c.getNcContainerMem() != null) {
        ncMem = Integer.parseInt(c.getNcContainerMem());
    } else {
        ncMem = CC_MEMORY_MBS_DEFAULT;
    }
    //request CC
    int numNodes = 0;
    ContainerRequest ccAsk = hostToRequest(cC.getClusterIp(), true);
    resourceManager.addContainerRequest(ccAsk);
    LOG.info("Asked for CC: " + Arrays.toString(ccAsk.getNodes().toArray()));
    numNodes++;
    //now we wait to be given the CC before starting the NCs...
    //we will wait a minute.
    int deathClock = 60;
    while (ccUp.get() == false && deathClock > 0) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException ex) {
            LOG.debug(ex);
        }
        --deathClock;
    }
    if (deathClock == 0 && ccUp.get() == false) {
        throw new YarnException("Couldn't allocate container for CC. Abort!");
    }
    LOG.info("Waiting for CC process to start");
    // is there a good way to do this? maybe try opening a socket to it...
    try {
        Thread.sleep(10000);
    } catch (InterruptedException ex) {
        LOG.debug(ex);
    }
    //request NCs
    for (Node n : c.getNode()) {
        resourceManager.addContainerRequest(hostToRequest(n.getClusterIp(), false));
        LOG.info("Asked for NC: " + n.getClusterIp());
        numNodes++;
        synchronized (pendingNCs) {
            pendingNCs.add(n);
        }
    }
    LOG.info("Requested all NCs and CCs. Wait for things to settle!");
    numRequestedContainers.set(numNodes);
    numTotalContainers = numNodes;
    doneAllocating = true;
}

Example 4 with Node

use of org.apache.asterix.event.schema.yarnCluster.Node in project asterixdb by apache.

the class AsterixApplicationMaster method containerToNode.

/**
     * Attempts to find the Node in the Cluster Description that matches this container
     *
     * @param c
     *            The container to resolve
     * @return The node this container corresponds to
     * @throws java.net.UnknownHostException
     *             if the container isn't present in the description
     */
Node containerToNode(Container c, Cluster cl) throws UnknownHostException {
    String containerHost = c.getNodeId().getHost();
    InetAddress containerIp = InetAddress.getByName(containerHost);
    LOG.info("Resolved Container IP: " + containerIp);
    for (Node node : cl.getNode()) {
        InetAddress nodeIp = InetAddress.getByName(node.getClusterIp());
        LOG.info(nodeIp + "?=" + containerIp);
        if (nodeIp.equals(containerIp)) {
            return node;
        }
    }
    //if we find nothing, this is bad...
    throw new java.net.UnknownHostException("Could not resolve container" + containerHost + " to node");
}