Examples with SuppressForbidden org.apache.lucene.util.SuppressForbidden used on opensource projects

Example 1 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class PrintTaxonomyStats method main.

/** Command-line tool. */
@SuppressForbidden(reason = "System.out required: command line tool")
public static void main(String[] args) throws IOException {
    boolean printTree = false;
    String path = null;
    for (int i = 0; i < args.length; i++) {
        if (args[i].equals("-printTree")) {
            printTree = true;
        } else {
            path = args[i];
        }
    }
    if (args.length != (printTree ? 2 : 1)) {
        System.out.println("\nUsage: java -classpath ... org.apache.lucene.facet.util.PrintTaxonomyStats [-printTree] /path/to/taxononmy/index\n");
        System.exit(1);
    }
    Directory dir = FSDirectory.open(Paths.get(path));
    TaxonomyReader r = new DirectoryTaxonomyReader(dir);
    printStats(r, System.out, printTree);
    r.close();
    dir.close();
}

Example 2 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class TestStressNRTReplication method startNode.

/** Launches a child "server" (separate JVM), which is either primary or replica node */
@SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")
NodeProcess startNode(final int id, Path indexPath, boolean isPrimary, long forcePrimaryVersion) throws IOException {
    nodeTimeStamps[id] = System.nanoTime();
    List<String> cmd = new ArrayList<>();
    NodeProcess curPrimary = primary;
    cmd.add(System.getProperty("java.home") + System.getProperty("file.separator") + "bin" + System.getProperty("file.separator") + "java");
    cmd.add("-Xmx512m");
    if (curPrimary != null) {
        cmd.add("-Dtests.nrtreplication.primaryTCPPort=" + curPrimary.tcpPort);
    } else if (isPrimary == false) {
        // We cannot start a replica when there is no primary:
        return null;
    }
    // This is very costly (takes more time to check than it did to index); we do this ourselves in the end instead of each time a replica
    // is restarted:
    // cmd.add("-Dtests.nrtreplication.checkonclose=true");
    cmd.add("-Dtests.nrtreplication.node=true");
    cmd.add("-Dtests.nrtreplication.nodeid=" + id);
    cmd.add("-Dtests.nrtreplication.startNS=" + Node.globalStartNS);
    cmd.add("-Dtests.nrtreplication.indexpath=" + indexPath);
    if (isPrimary) {
        cmd.add("-Dtests.nrtreplication.isPrimary=true");
        cmd.add("-Dtests.nrtreplication.forcePrimaryVersion=" + forcePrimaryVersion);
        if (DO_CRASH_PRIMARY) {
            cmd.add("-Dtests.nrtreplication.doRandomCrash=true");
        }
        if (DO_CLOSE_PRIMARY) {
            cmd.add("-Dtests.nrtreplication.doRandomClose=true");
        }
    } else {
        if (DO_CRASH_REPLICA) {
            cmd.add("-Dtests.nrtreplication.doRandomCrash=true");
        }
        if (DO_CLOSE_REPLICA) {
            cmd.add("-Dtests.nrtreplication.doRandomClose=true");
        }
    }
    if (DO_BIT_FLIPS_DURING_COPY) {
        cmd.add("-Dtests.nrtreplication.doFlipBitsDuringCopy=true");
    }
    long myPrimaryGen = primaryGen;
    cmd.add("-Dtests.nrtreplication.primaryGen=" + myPrimaryGen);
    // Mixin our own counter because this is called from a fresh thread which means the seed otherwise isn't changing each time we spawn a
    // new node:
    long seed = random().nextLong() * nodeStartCounter.incrementAndGet();
    cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed));
    cmd.add("-ea");
    cmd.add("-cp");
    cmd.add(System.getProperty("java.class.path"));
    cmd.add("org.junit.runner.JUnitCore");
    cmd.add(getClass().getName().replace(getClass().getSimpleName(), "SimpleServer"));
    Writer childLog;
    if (SEPARATE_CHILD_OUTPUT) {
        Path childOut = childTempDir.resolve(id + ".log");
        message("logging to " + childOut);
        childLog = Files.newBufferedWriter(childOut, StandardCharsets.UTF_8, StandardOpenOption.APPEND, StandardOpenOption.CREATE);
        childLog.write("\n\nSTART NEW CHILD:\n");
    } else {
        childLog = null;
    }
    //message("child process command: " + cmd);
    ProcessBuilder pb = new ProcessBuilder(cmd);
    pb.redirectErrorStream(true);
    // Important, so that the scary looking hs_err_<pid>.log appear under our test temp dir:
    pb.directory(childTempDir.toFile());
    Process p = pb.start();
    BufferedReader r;
    try {
        r = new BufferedReader(new InputStreamReader(p.getInputStream(), IOUtils.UTF_8));
    } catch (UnsupportedEncodingException uee) {
        throw new RuntimeException(uee);
    }
    int tcpPort = -1;
    long initCommitVersion = -1;
    long initInfosVersion = -1;
    Pattern logTimeStart = Pattern.compile("^[0-9\\.]+s .*");
    boolean willCrash = false;
    while (true) {
        String l = r.readLine();
        if (l == null) {
            message("top: node=" + id + " failed to start");
            try {
                p.waitFor();
            } catch (InterruptedException ie) {
                throw new RuntimeException(ie);
            }
            message("exit value=" + p.exitValue());
            if (p.exitValue() == 0) {
                message("zero exit status; assuming failed to remove segments_N; skipping");
                return null;
            }
            // Hackity hack, in case primary crashed/closed and we haven't noticed (reaped the process) yet:
            if (isPrimary == false) {
                for (int i = 0; i < 100; i++) {
                    NodeProcess primary2 = primary;
                    if (primaryGen != myPrimaryGen || primary2 == null || primary2.nodeIsClosing.get()) {
                        // OK: primary crashed while we were trying to start, so it's expected/allowed that we could not start the replica:
                        message("primary crashed/closed while replica R" + id + " tried to start; skipping");
                        return null;
                    } else {
                        try {
                            Thread.sleep(10);
                        } catch (InterruptedException ie) {
                            throw new ThreadInterruptedException(ie);
                        }
                    }
                }
            }
            // Should fail the test:
            message("top: now fail test replica R" + id + " failed to start");
            failed.set(true);
            throw new RuntimeException("replica R" + id + " failed to start");
        }
        if (childLog != null) {
            childLog.write(l);
            childLog.write("\n");
            childLog.flush();
        } else if (logTimeStart.matcher(l).matches()) {
            // Already a well-formed log output:
            System.out.println(l);
        } else {
            message(l);
        }
        if (l.startsWith("PORT: ")) {
            tcpPort = Integer.parseInt(l.substring(6).trim());
        } else if (l.startsWith("COMMIT VERSION: ")) {
            initCommitVersion = Integer.parseInt(l.substring(16).trim());
        } else if (l.startsWith("INFOS VERSION: ")) {
            initInfosVersion = Integer.parseInt(l.substring(15).trim());
        } else if (l.contains("will crash after")) {
            willCrash = true;
        } else if (l.startsWith("NODE STARTED")) {
            break;
        }
    }
    final boolean finalWillCrash = willCrash;
    final AtomicBoolean nodeIsClosing = new AtomicBoolean();
    // Baby sits the child process, pulling its stdout and printing to our stdout, calling nodeClosed once it exits:
    Thread pumper = ThreadPumper.start(new Runnable() {

        @Override
        public void run() {
            message("now wait for process " + p);
            try {
                p.waitFor();
            } catch (Throwable t) {
                throw new RuntimeException(t);
            }
            message("done wait for process " + p);
            int exitValue = p.exitValue();
            message("exit value=" + exitValue + " willCrash=" + finalWillCrash);
            if (childLog != null) {
                try {
                    childLog.write("process done; exitValue=" + exitValue + "\n");
                    childLog.close();
                } catch (IOException ioe) {
                    throw new RuntimeException(ioe);
                }
            }
            if (exitValue != 0 && finalWillCrash == false && crashingNodes.remove(id) == false) {
                // should fail test
                failed.set(true);
                if (childLog != null) {
                    throw new RuntimeException("node " + id + " process had unexpected non-zero exit status=" + exitValue + "; see " + childLog + " for details");
                } else {
                    throw new RuntimeException("node " + id + " process had unexpected non-zero exit status=" + exitValue);
                }
            }
            nodeClosed(id);
        }
    }, r, System.out, childLog, nodeIsClosing);
    pumper.setName("pump" + id);
    message("top: node=" + id + " started at tcpPort=" + tcpPort + " initCommitVersion=" + initCommitVersion + " initInfosVersion=" + initInfosVersion);
    return new NodeProcess(p, id, tcpPort, pumper, isPrimary, initCommitVersion, initInfosVersion, nodeIsClosing);
}

Example 3 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class TestNRTReplication method startNode.

/** Launches a child "server" (separate JVM), which is either primary or replica node */
@SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")
private NodeProcess startNode(int primaryTCPPort, final int id, Path indexPath, long forcePrimaryVersion, boolean willCrash) throws IOException {
    List<String> cmd = new ArrayList<>();
    cmd.add(System.getProperty("java.home") + System.getProperty("file.separator") + "bin" + System.getProperty("file.separator") + "java");
    cmd.add("-Xmx512m");
    long myPrimaryGen;
    if (primaryTCPPort != -1) {
        // I am a replica
        cmd.add("-Dtests.nrtreplication.primaryTCPPort=" + primaryTCPPort);
        myPrimaryGen = lastPrimaryGen;
    } else {
        myPrimaryGen = nextPrimaryGen++;
        lastPrimaryGen = myPrimaryGen;
    }
    cmd.add("-Dtests.nrtreplication.primaryGen=" + myPrimaryGen);
    cmd.add("-Dtests.nrtreplication.closeorcrash=false");
    cmd.add("-Dtests.nrtreplication.node=true");
    cmd.add("-Dtests.nrtreplication.nodeid=" + id);
    cmd.add("-Dtests.nrtreplication.startNS=" + Node.globalStartNS);
    cmd.add("-Dtests.nrtreplication.indexpath=" + indexPath);
    cmd.add("-Dtests.nrtreplication.checkonclose=true");
    if (primaryTCPPort == -1) {
        // We are the primary node
        cmd.add("-Dtests.nrtreplication.isPrimary=true");
        cmd.add("-Dtests.nrtreplication.forcePrimaryVersion=" + forcePrimaryVersion);
    }
    // Mixin our own counter because this is called from a fresh thread which means the seed otherwise isn't changing each time we spawn a
    // new node:
    long seed = random().nextLong() * nodeStartCounter.incrementAndGet();
    cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed));
    cmd.add("-ea");
    cmd.add("-cp");
    cmd.add(System.getProperty("java.class.path"));
    cmd.add("org.junit.runner.JUnitCore");
    cmd.add(getClass().getName().replace(getClass().getSimpleName(), "SimpleServer"));
    message("child process command: " + cmd);
    ProcessBuilder pb = new ProcessBuilder(cmd);
    pb.redirectErrorStream(true);
    // Important, so that the scary looking hs_err_<pid>.log appear under our test temp dir:
    pb.directory(childTempDir.toFile());
    Process p = pb.start();
    BufferedReader r;
    try {
        r = new BufferedReader(new InputStreamReader(p.getInputStream(), IOUtils.UTF_8));
    } catch (UnsupportedEncodingException uee) {
        throw new RuntimeException(uee);
    }
    int tcpPort = -1;
    long initCommitVersion = -1;
    long initInfosVersion = -1;
    Pattern logTimeStart = Pattern.compile("^[0-9\\.]+s .*");
    boolean sawExistingSegmentsFile = false;
    while (true) {
        String l = r.readLine();
        if (l == null) {
            message("top: node=" + id + " failed to start");
            try {
                p.waitFor();
            } catch (InterruptedException ie) {
                throw new RuntimeException(ie);
            }
            message("exit value=" + p.exitValue());
            message("top: now fail test replica R" + id + " failed to start");
            throw new RuntimeException("replica R" + id + " failed to start");
        }
        if (logTimeStart.matcher(l).matches()) {
            // Already a well-formed log output:
            System.out.println(l);
        } else {
            message(l);
        }
        if (l.startsWith("PORT: ")) {
            tcpPort = Integer.parseInt(l.substring(6).trim());
        } else if (l.startsWith("COMMIT VERSION: ")) {
            initCommitVersion = Integer.parseInt(l.substring(16).trim());
        } else if (l.startsWith("INFOS VERSION: ")) {
            initInfosVersion = Integer.parseInt(l.substring(15).trim());
        } else if (l.contains("will crash after")) {
            willCrash = true;
        } else if (l.startsWith("NODE STARTED")) {
            break;
        } else if (l.contains("replica cannot start: existing segments file=")) {
            sawExistingSegmentsFile = true;
        }
    }
    final boolean finalWillCrash = willCrash;
    // Baby sits the child process, pulling its stdout and printing to our stdout:
    AtomicBoolean nodeClosing = new AtomicBoolean();
    Thread pumper = ThreadPumper.start(new Runnable() {

        @Override
        public void run() {
            message("now wait for process " + p);
            try {
                p.waitFor();
            } catch (Throwable t) {
                throw new RuntimeException(t);
            }
            message("done wait for process " + p);
            int exitValue = p.exitValue();
            message("exit value=" + exitValue + " willCrash=" + finalWillCrash);
            if (exitValue != 0 && finalWillCrash == false) {
                // should fail test
                throw new RuntimeException("node " + id + " process had unexpected non-zero exit status=" + exitValue);
            }
        }
    }, r, System.out, null, nodeClosing);
    pumper.setName("pump" + id);
    message("top: node=" + id + " started at tcpPort=" + tcpPort + " initCommitVersion=" + initCommitVersion + " initInfosVersion=" + initInfosVersion);
    return new NodeProcess(p, id, tcpPort, pumper, primaryTCPPort == -1, initCommitVersion, initInfosVersion, nodeClosing);
}

Example 4 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class IndexSplitter method listSegments.

@SuppressForbidden(reason = "System.out required: command line tool")
public void listSegments() throws IOException {
    DecimalFormat formatter = new DecimalFormat("###,###.###", DecimalFormatSymbols.getInstance(Locale.ROOT));
    for (int x = 0; x < infos.size(); x++) {
        SegmentCommitInfo info = infos.info(x);
        String sizeStr = formatter.format(info.sizeInBytes());
        System.out.println(info.info.name + " " + sizeStr);
    }
}

Example 5 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class HighFreqTerms method main.

@SuppressForbidden(reason = "System.out required: command line tool")
public static void main(String[] args) throws Exception {
    String field = null;
    int numTerms = DEFAULT_NUMTERMS;
    if (args.length == 0 || args.length > 4) {
        usage();
        System.exit(1);
    }
    Directory dir = FSDirectory.open(Paths.get(args[0]));
    Comparator<TermStats> comparator = new DocFreqComparator();
    for (int i = 1; i < args.length; i++) {
        if (args[i].equals("-t")) {
            comparator = new TotalTermFreqComparator();
        } else {
            try {
                numTerms = Integer.parseInt(args[i]);
            } catch (NumberFormatException e) {
                field = args[i];
            }
        }
    }
    IndexReader reader = DirectoryReader.open(dir);
    TermStats[] terms = getHighFreqTerms(reader, numTerms, field, comparator);
    for (int i = 0; i < terms.length; i++) {
        System.out.printf(Locale.ROOT, "%s:%s \t totalTF = %,d \t docFreq = %,d \n", terms[i].field, terms[i].termtext.utf8ToString(), terms[i].totalTermFreq, terms[i].docFreq);
    }
    reader.close();
}