Examples with CommandLine org.voltdb.utils.CommandLine used on opensource projects

Example 1 with CommandLine

use of org.voltdb.utils.CommandLine in project voltdb by VoltDB.

the class LocalCluster method startLocalServer.

private void startLocalServer(int hostId, boolean clearLocalDataDirectories, StartAction action) throws IOException {
    // Generate a new root for the in-process server if clearing directories.
    File subroot = null;
    if (!isNewCli) {
        try {
            if (m_filePrefix != null) {
                subroot = m_filePrefix;
                m_subRoots.add(subroot);
            } else if (clearLocalDataDirectories) {
                subroot = VoltFile.initNewSubrootForThisProcess();
                m_subRoots.add(subroot);
            } else {
                if (m_subRoots.size() <= hostId) {
                    m_subRoots.add(VoltFile.initNewSubrootForThisProcess());
                }
                subroot = m_subRoots.get(hostId);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    // Make the local Configuration object...
    CommandLine cmdln = (templateCmdLine.makeCopy());
    cmdln.startCommand(action);
    cmdln.setJavaProperty(clusterHostIdProperty, String.valueOf(hostId));
    if (this.m_additionalProcessEnv != null) {
        for (String name : this.m_additionalProcessEnv.keySet()) {
            cmdln.setJavaProperty(name, this.m_additionalProcessEnv.get(name));
        }
    }
    if (!isNewCli) {
        cmdln.voltFilePrefix(subroot.getPath());
    }
    cmdln.internalPort(internalPortGenerator.nextInternalPort(hostId));
    cmdln.coordinators(internalPortGenerator.getCoordinators());
    cmdln.port(portGenerator.nextClient());
    cmdln.adminPort(portGenerator.nextAdmin());
    cmdln.zkport(portGenerator.nextZkPort());
    cmdln.httpPort(portGenerator.nextHttp());
    // replication port and its two automatic followers.
    cmdln.drAgentStartPort(m_replicationPort != -1 ? m_replicationPort : portGenerator.nextReplicationPort());
    portGenerator.nextReplicationPort();
    portGenerator.nextReplicationPort();
    if (m_target == BackendTarget.NATIVE_EE_VALGRIND_IPC) {
        EEProcess proc = m_eeProcs.get(hostId);
        assert (proc != null);
        cmdln.m_ipcPort = proc.port();
    }
    if (m_target == BackendTarget.NATIVE_EE_IPC) {
        cmdln.m_ipcPort = portGenerator.next();
    }
    if ((m_versionOverrides != null) && (m_versionOverrides.length > hostId)) {
        assert (m_versionOverrides[hostId] != null);
        assert (m_versionCheckRegexOverrides[hostId] != null);
        cmdln.m_versionStringOverrideForTest = m_versionOverrides[hostId];
        cmdln.m_versionCompatibilityRegexOverrideForTest = m_versionCheckRegexOverrides[hostId];
        if ((m_buildStringOverrides != null) && (m_buildStringOverrides.length > hostId)) {
            assert (m_buildStringOverrides[hostId] != null);
            cmdln.m_buildStringOverrideForTest = m_buildStringOverrides[hostId];
        }
    }
    if ((m_modeOverrides != null) && (m_modeOverrides.length > hostId)) {
        assert (m_modeOverrides[hostId] != null);
        cmdln.m_modeOverrideForTest = m_modeOverrides[hostId];
        cmdln.m_isPaused = true;
    }
    if ((m_sitesperhostOverrides != null) && (m_sitesperhostOverrides.size() > hostId)) {
        assert (m_sitesperhostOverrides.containsKey(hostId));
        cmdln.m_sitesperhost = m_sitesperhostOverrides.get(hostId);
    }
    // for debug, dump the command line to a unique file.
    // cmdln.dumpToFile("/Users/rbetts/cmd_" + Integer.toString(portGenerator.next()));
    m_cluster.add(null);
    m_pipes.add(null);
    m_cmdLines.add(cmdln);
    if (isNewCli) {
        cmdln.m_startAction = StartAction.PROBE;
        cmdln.enableAdd(action == StartAction.JOIN);
        cmdln.m_hostCount = m_hostCount;
        String hostIdStr = cmdln.getJavaProperty(clusterHostIdProperty);
        String root = m_hostRoots.get(hostIdStr);
        //For new CLI dont pass deployment for probe.
        cmdln.pathToDeployment(null);
        cmdln.voltdbRoot(root + File.separator + Constants.DBROOT);
    }
    m_localServer = new ServerThread(cmdln);
    if (m_usesStagedSchema) {
        // ServerThread sets this to true, always - override with our desired behavior.
        // Only do this for staged schema tests - preserve old behavior for others.
        cmdln.setForceVoltdbCreate(clearLocalDataDirectories);
    }
    m_localServer.start();
}

Example 2 with CommandLine

use of org.voltdb.utils.CommandLine in project voltdb by VoltDB.

the class LocalCluster method startOne.

void startOne(int hostId, boolean clearLocalDataDirectories, StartAction startAction, boolean waitForReady, String placementGroup) throws IOException {
    PipeToFile ptf = null;
    CommandLine cmdln = (templateCmdLine.makeCopy());
    cmdln.setJavaProperty(clusterHostIdProperty, String.valueOf(hostId));
    if (isNewCli) {
        cmdln.m_startAction = StartAction.PROBE;
        cmdln.enableAdd(startAction == StartAction.JOIN);
        cmdln.hostCount(m_hostCount);
        String hostIdStr = cmdln.getJavaProperty(clusterHostIdProperty);
        String root = m_hostRoots.get(hostIdStr);
        //For new CLI dont pass deployment for probe.
        cmdln.voltdbRoot(root);
        cmdln.pathToDeployment(null);
        cmdln.setForceVoltdbCreate(clearLocalDataDirectories);
    }
    if (this.m_additionalProcessEnv != null) {
        for (String name : this.m_additionalProcessEnv.keySet()) {
            cmdln.setJavaProperty(name, this.m_additionalProcessEnv.get(name));
        }
    }
    try {
        cmdln.internalPort(internalPortGenerator.nextInternalPort(hostId));
        cmdln.coordinators(internalPortGenerator.getCoordinators());
        if (m_replicationPort != -1) {
            int index = m_hasLocalServer ? hostId + 1 : hostId;
            cmdln.drAgentStartPort(m_replicationPort + index);
        } else {
            // set the dragent port. it uses the start value and
            // the next two sequential port numbers - so burn those two.
            cmdln.drAgentStartPort(portGenerator.nextReplicationPort());
            portGenerator.next();
            portGenerator.next();
        }
        // add the ipc ports
        if (m_target == BackendTarget.NATIVE_EE_IPC) {
            // set 1 port for the EE process
            cmdln.ipcPort(portGenerator.next());
        }
        if (m_target == BackendTarget.NATIVE_EE_VALGRIND_IPC) {
            EEProcess proc = m_eeProcs.get(hostId);
            assert (proc != null);
            cmdln.m_ipcPort = proc.port();
        }
        cmdln.port(portGenerator.nextClient());
        cmdln.adminPort(portGenerator.nextAdmin());
        if (cmdln.m_httpPort != Constants.HTTP_PORT_DISABLED)
            cmdln.httpPort(portGenerator.nextHttp());
        cmdln.timestampSalt(getRandomTimestampSalt());
        cmdln.setPlacementGroup(placementGroup);
        if (m_debug) {
            cmdln.debugPort(portGenerator.next());
        }
        cmdln.zkport(portGenerator.nextZkPort());
        if (!isNewCli && startAction == StartAction.JOIN) {
            cmdln.startCommand(startAction);
            int portNoToRejoin = m_cmdLines.get(0).internalPort();
            cmdln.leader(":" + portNoToRejoin);
            cmdln.enableAdd(true);
        }
        // If local directories are being cleared
        // generate a new subroot, otherwise reuse the existing directory
        File subroot = null;
        if (!isNewCli) {
            if (m_filePrefix != null) {
                subroot = m_filePrefix;
                m_subRoots.add(subroot);
            } else if (clearLocalDataDirectories) {
                subroot = VoltFile.getNewSubroot();
                m_subRoots.add(subroot);
            } else {
                if (m_subRoots.size() <= hostId) {
                    m_subRoots.add(VoltFile.getNewSubroot());
                }
                subroot = m_subRoots.get(hostId);
            }
            cmdln.voltFilePrefix(subroot.getPath());
            cmdln.voltRoot(subroot.getPath() + File.separator + m_voltdbroot);
        }
        if ((m_versionOverrides != null) && (m_versionOverrides.length > hostId)) {
            assert (m_versionOverrides[hostId] != null);
            assert (m_versionCheckRegexOverrides[hostId] != null);
            cmdln.m_versionStringOverrideForTest = m_versionOverrides[hostId];
            cmdln.m_versionCompatibilityRegexOverrideForTest = m_versionCheckRegexOverrides[hostId];
            if ((m_buildStringOverrides != null) && (m_buildStringOverrides.length > hostId)) {
                assert (m_buildStringOverrides[hostId] != null);
                cmdln.m_buildStringOverrideForTest = m_buildStringOverrides[hostId];
            }
        }
        if ((m_modeOverrides != null) && (m_modeOverrides.length > hostId)) {
            assert (m_modeOverrides[hostId] != null);
            cmdln.m_modeOverrideForTest = m_modeOverrides[hostId];
        }
        if ((m_sitesperhostOverrides != null) && (m_sitesperhostOverrides.size() > hostId)) {
            assert (m_sitesperhostOverrides.containsKey(hostId));
            cmdln.m_sitesperhost = m_sitesperhostOverrides.get(hostId);
        }
        cmdln.setMissingHostCount(m_missingHostCount);
        m_cmdLines.add(cmdln);
        m_procBuilder.command().clear();
        List<String> cmdlnList = cmdln.createCommandLine();
        String cmdLineFull = "Start cmd host=" + String.valueOf(hostId) + " :";
        for (String element : cmdlnList) {
            assert (element != null);
            cmdLineFull += " " + element;
        }
        log.info(cmdLineFull);
        System.out.println(cmdLineFull);
        m_procBuilder.command().addAll(cmdlnList);
        // write output to obj/release/testoutput/<test name>-n.txt
        // this may need to be more unique? Also very useful to just
        // set this to a hardcoded path and use "tail -f" to debug.
        String testoutputdir = cmdln.buildDir() + File.separator + "testoutput";
        System.out.println("Process output will be redirected to: " + testoutputdir);
        // make sure the directory exists
        File dir = new File(testoutputdir);
        if (dir.exists()) {
            assert (dir.isDirectory());
        } else {
            boolean status = dir.mkdirs();
            assert (status);
        }
        File dirFile = new VoltFile(testoutputdir);
        if (dirFile.listFiles() != null) {
            for (File f : dirFile.listFiles()) {
                if (f.getName().startsWith(getName() + "-" + hostId)) {
                    f.delete();
                }
            }
        }
        Process proc = m_procBuilder.start();
        m_cluster.add(proc);
        String fileName = testoutputdir + File.separator + "LC-" + getFileName() + "-" + m_clusterId + "-" + hostId + "-" + "idx" + String.valueOf(perLocalClusterExtProcessIndex++) + ".txt";
        System.out.println("Process output can be found in: " + fileName);
        ptf = new PipeToFile(fileName, proc.getInputStream(), PipeToFile.m_initToken, false, proc);
        m_pipes.add(ptf);
        ptf.setName("ClusterPipe:" + String.valueOf(hostId));
        ptf.start();
    } catch (IOException ex) {
        log.error("Failed to start cluster process:" + ex.getMessage(), ex);
        assert (false);
    }
    if (waitForReady && (startAction == StartAction.JOIN || startAction == StartAction.PROBE || startAction == StartAction.REJOIN)) {
        waitOnPTFReady(ptf, true, System.currentTimeMillis(), System.currentTimeMillis(), hostId);
    }
    if (hostId > (m_hostCount - 1)) {
        m_hostCount++;
        //Host count changed, should recompile
        this.m_compiled = false;
    }
}

Example 3 with CommandLine

use of org.voltdb.utils.CommandLine in project voltdb by VoltDB.

the class LocalCluster method setPortsFromConfig.

public void setPortsFromConfig(int hostId, VoltDB.Configuration config) {
    CommandLine cl = m_cmdLines.get(hostId);
    assert (cl != null);
    cl.m_port = config.m_port;
    cl.m_adminPort = config.m_adminPort;
    cl.m_zkInterface = config.m_zkInterface;
    cl.m_internalPort = config.m_internalPort;
    cl.m_leader = config.m_leader;
    cl.m_coordinators = ImmutableSortedSet.copyOf(config.m_coordinators);
}

Example 4 with CommandLine

use of org.voltdb.utils.CommandLine in project voltdb by VoltDB.

the class LocalCluster method initOne.

private void initOne(int hostId, boolean clearLocalDataDirectories) throws IOException {
    PipeToFile ptf = null;
    CommandLine cmdln = (templateCmdLine.makeCopy());
    cmdln.setJavaProperty(clusterHostIdProperty, String.valueOf(hostId));
    if (this.m_additionalProcessEnv != null) {
        for (String name : this.m_additionalProcessEnv.keySet()) {
            cmdln.setJavaProperty(name, this.m_additionalProcessEnv.get(name));
        }
    }
    File root = null;
    try {
        //If clear clean VoltFile.getServerSpecificRoot(String.valueOf(hostId))
        root = VoltFile.getServerSpecificRoot(String.valueOf(hostId), clearLocalDataDirectories);
        assert (root.getName().equals(Constants.DBROOT) == false) : root.getAbsolutePath();
        cmdln = cmdln.voltdbRoot(new File(root, Constants.DBROOT));
        cmdln = cmdln.startCommand(StartAction.INITIALIZE);
        if (clearLocalDataDirectories) {
            cmdln.setForceVoltdbCreate(true);
        } else {
            cmdln.setForceVoltdbCreate(false);
        }
        if (new Integer(hostId).equals(m_mismatchNode)) {
            assert m_usesStagedSchema;
            cmdln.m_userSchema = m_mismatchSchema == null ? null : VoltProjectBuilder.createFileForSchema(m_mismatchSchema);
        }
        m_procBuilder.command().clear();
        List<String> cmdlnList = cmdln.createCommandLine();
        String cmdLineFull = "Init cmd host=" + String.valueOf(hostId) + " :";
        for (String element : cmdlnList) {
            assert (element != null);
            cmdLineFull += " " + element;
        }
        log.info(cmdLineFull);
        m_procBuilder.command().addAll(cmdlnList);
        // write output to obj/release/testoutput/<test name>-n.txt
        // this may need to be more unique? Also very useful to just
        // set this to a hardcoded path and use "tail -f" to debug.
        String testoutputdir = cmdln.buildDir() + File.separator + "testoutput";
        System.out.println("Process output will be redirected to: " + testoutputdir);
        // make sure the directory exists
        File dir = new File(testoutputdir);
        if (dir.exists()) {
            assert (dir.isDirectory());
        } else {
            boolean status = dir.mkdirs();
            assert (status);
        }
        File dirFile = new VoltFile(testoutputdir);
        if (dirFile.listFiles() != null) {
            for (File f : dirFile.listFiles()) {
                if (f.getName().startsWith(getName() + "-" + hostId)) {
                    f.delete();
                }
            }
        }
        Process proc = m_procBuilder.start();
        String fileName = testoutputdir + File.separator + "LC-" + getFileName() + "-" + m_clusterId + "-init-" + hostId + "-" + "idx" + String.valueOf(perLocalClusterExtProcessIndex++) + ".txt";
        System.out.println("Process output can be found in: " + fileName);
        ptf = new PipeToFile(fileName, proc.getInputStream(), String.valueOf(hostId), false, proc);
        ptf.setName("ClusterPipe:" + String.valueOf(hostId));
        ptf.start();
        proc.waitFor();
    } catch (IOException ex) {
        log.error("Failed to start cluster process:" + ex.getMessage(), ex);
        assert (false);
    } catch (InterruptedException ex) {
        log.error("Failed to start cluster process:" + ex.getMessage(), ex);
        assert (false);
    }
    if (root != null) {
        String hostIdStr = cmdln.getJavaProperty(clusterHostIdProperty);
        m_hostRoots.put(hostIdStr, root.getPath());
    }
}

Example 5 with CommandLine

use of org.voltdb.utils.CommandLine in project voltdb by VoltDB.

the class LocalCluster method recoverOne.

// Re-start a (dead) process. HostId is the enumberation of the host
// in the cluster (0, 1, ... hostCount-1) -- not an hsid, for example.
private boolean recoverOne(boolean logtime, long startTime, int hostId, Integer rejoinHostId, String rejoinHost, StartAction startAction) {
    // it this way originally probably eats kittens and hates cake.
    if (rejoinHostId == null || m_hasLocalServer) {
        rejoinHostId = 0;
    }
    if (isNewCli) {
        //If this is new CLI we use probe
        startAction = StartAction.PROBE;
    }
    int portNoToRejoin = m_cmdLines.get(rejoinHostId).internalPort();
    if (hostId == 0 && m_hasLocalServer) {
        templateCmdLine.leaderPort(portNoToRejoin);
        try {
            startLocalServer(rejoinHostId, false, startAction);
        } catch (IOException ioe) {
            throw new RuntimeException(ioe);
        }
        return true;
    }
    log.info("Rejoining " + hostId + " to hostID: " + rejoinHostId);
    // rebuild the EE proc set.
    if (templateCmdLine.target().isIPC && m_eeProcs.contains(hostId)) {
        EEProcess eeProc = m_eeProcs.get(hostId);
        File valgrindOutputFile = null;
        try {
            valgrindOutputFile = eeProc.waitForShutdown();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        failIfValgrindErrors(valgrindOutputFile);
    }
    if (templateCmdLine.target().isIPC) {
        String logfile = "LocalCluster_host_" + hostId + ".log";
        m_eeProcs.set(hostId, new EEProcess(templateCmdLine.target(), m_siteCount, logfile));
    }
    PipeToFile ptf = null;
    long start = 0;
    try {
        CommandLine rejoinCmdLn = m_cmdLines.get(hostId);
        rejoinCmdLn.setForceVoltdbCreate(false);
        // some tests need this
        rejoinCmdLn.javaProperties = templateCmdLine.javaProperties;
        rejoinCmdLn.setJavaProperty(clusterHostIdProperty, String.valueOf(hostId));
        rejoinCmdLn.startCommand(startAction);
        rejoinCmdLn.setJavaProperty(clusterHostIdProperty, String.valueOf(hostId));
        // Bump it to avoid collisions on rejoin.
        if (m_debug) {
            rejoinCmdLn.debugPort(portGenerator.next());
        }
        rejoinCmdLn.leader(rejoinHost + ":" + String.valueOf(portNoToRejoin));
        rejoinCmdLn.m_port = portGenerator.nextClient();
        rejoinCmdLn.m_adminPort = portGenerator.nextAdmin();
        rejoinCmdLn.m_httpPort = portGenerator.nextHttp();
        rejoinCmdLn.m_zkInterface = "127.0.0.1:" + portGenerator.next();
        rejoinCmdLn.m_internalPort = internalPortGenerator.nextInternalPort(hostId);
        rejoinCmdLn.m_coordinators = internalPortGenerator.getCoordinators();
        setPortsFromConfig(hostId, rejoinCmdLn);
        if (this.m_additionalProcessEnv != null) {
            for (String name : this.m_additionalProcessEnv.keySet()) {
                rejoinCmdLn.setJavaProperty(name, this.m_additionalProcessEnv.get(name));
            }
        }
        //rejoin can hotfix
        if ((m_versionOverrides != null) && (m_versionOverrides.length > hostId)) {
            assert (m_versionOverrides[hostId] != null);
            assert (m_versionCheckRegexOverrides[hostId] != null);
            rejoinCmdLn.m_versionStringOverrideForTest = m_versionOverrides[hostId];
            rejoinCmdLn.m_versionCompatibilityRegexOverrideForTest = m_versionCheckRegexOverrides[hostId];
            if ((m_buildStringOverrides != null) && (m_buildStringOverrides.length > hostId)) {
                assert (m_buildStringOverrides[hostId] != null);
                rejoinCmdLn.m_buildStringOverrideForTest = m_buildStringOverrides[hostId];
            }
        }
        //Rejoin mixed sitesperhost
        if ((m_sitesperhostOverrides != null) && (m_sitesperhostOverrides.size() > hostId)) {
            assert (m_sitesperhostOverrides.containsKey(hostId));
            rejoinCmdLn.m_sitesperhost = m_sitesperhostOverrides.get(hostId);
        }
        List<String> rejoinCmdLnStr = rejoinCmdLn.createCommandLine();
        String cmdLineFull = "Rejoin cmd line:";
        for (String element : rejoinCmdLnStr) {
            cmdLineFull += " " + element;
        }
        log.info(cmdLineFull);
        m_procBuilder.command().clear();
        m_procBuilder.command().addAll(rejoinCmdLnStr);
        Process proc = m_procBuilder.start();
        start = System.currentTimeMillis();
        // write output to obj/release/testoutput/<test name>-n.txt
        // this may need to be more unique? Also very useful to just
        // set this to a hardcoded path and use "tail -f" to debug.
        String testoutputdir = rejoinCmdLn.buildDir() + File.separator + "testoutput";
        // make sure the directory exists
        File dir = new File(testoutputdir);
        if (dir.exists()) {
            assert (dir.isDirectory());
        } else {
            boolean status = dir.mkdirs();
            assert (status);
        }
        ptf = new PipeToFile(testoutputdir + File.separator + "LC-" + getFileName() + "-" + hostId + "-" + "idx" + String.valueOf(perLocalClusterExtProcessIndex++) + ".rejoined.txt", proc.getInputStream(), PipeToFile.m_initToken, true, proc);
        synchronized (this) {
            m_pipes.set(hostId, ptf);
            // replace the existing dead proc
            m_cluster.set(hostId, proc);
            m_cmdLines.set(hostId, rejoinCmdLn);
        }
        Thread t = new Thread(ptf);
        t.setName("ClusterPipe:" + String.valueOf(hostId));
        t.start();
    } catch (IOException ex) {
        log.error("Failed to start recovering cluster process:" + ex.getMessage(), ex);
        assert (false);
    }
    m_running = true;
    return waitOnPTFReady(ptf, logtime, startTime, start, hostId);
}