Examples with SuppressForbidden org.apache.lucene.util.SuppressForbidden used on opensource projects

Example 6 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class LockStressTest method main.

@SuppressForbidden(reason = "System.out required: command line tool")
@SuppressWarnings("try")
public static void main(String[] args) throws Exception {
    if (args.length != 7) {
        System.out.println("Usage: java org.apache.lucene.store.LockStressTest myID verifierHost verifierPort lockFactoryClassName lockDirName sleepTimeMS count\n" + "\n" + "  myID = int from 0 .. 255 (should be unique for test process)\n" + "  verifierHost = hostname that LockVerifyServer is listening on\n" + "  verifierPort = port that LockVerifyServer is listening on\n" + "  lockFactoryClassName = primary FSLockFactory class that we will use\n" + "  lockDirName = path to the lock directory\n" + "  sleepTimeMS = milliseconds to pause betweeen each lock obtain/release\n" + "  count = number of locking tries\n" + "\n" + "You should run multiple instances of this process, each with its own\n" + "unique ID, and each pointing to the same lock directory, to verify\n" + "that locking is working correctly.\n" + "\n" + "Make sure you are first running LockVerifyServer.");
        System.exit(1);
    }
    int arg = 0;
    final int myID = Integer.parseInt(args[arg++]);
    if (myID < 0 || myID > 255) {
        System.out.println("myID must be a unique int 0..255");
        System.exit(1);
    }
    final String verifierHost = args[arg++];
    final int verifierPort = Integer.parseInt(args[arg++]);
    final String lockFactoryClassName = args[arg++];
    final Path lockDirPath = Paths.get(args[arg++]);
    final int sleepTimeMS = Integer.parseInt(args[arg++]);
    final int count = Integer.parseInt(args[arg++]);
    final LockFactory lockFactory = getNewLockFactory(lockFactoryClassName);
    // we test the lock factory directly, so we don't need it on the directory itsself (the directory is just for testing)
    final FSDirectory lockDir = new SimpleFSDirectory(lockDirPath, NoLockFactory.INSTANCE);
    final InetSocketAddress addr = new InetSocketAddress(verifierHost, verifierPort);
    System.out.println("Connecting to server " + addr + " and registering as client " + myID + "...");
    try (Socket socket = new Socket()) {
        socket.setReuseAddress(true);
        socket.connect(addr, 500);
        final OutputStream out = socket.getOutputStream();
        final InputStream in = socket.getInputStream();
        out.write(myID);
        out.flush();
        LockFactory verifyLF = new VerifyingLockFactory(lockFactory, in, out);
        final Random rnd = new Random();
        // wait for starting gun
        if (in.read() != 43) {
            throw new IOException("Protocol violation");
        }
        for (int i = 0; i < count; i++) {
            try (final Lock l = verifyLF.obtainLock(lockDir, LOCK_FILE_NAME)) {
                if (rnd.nextInt(10) == 0) {
                    if (rnd.nextBoolean()) {
                        verifyLF = new VerifyingLockFactory(getNewLockFactory(lockFactoryClassName), in, out);
                    }
                    try (final Lock secondLock = verifyLF.obtainLock(lockDir, LOCK_FILE_NAME)) {
                        throw new IOException("Double obtain");
                    } catch (LockObtainFailedException loe) {
                    // pass
                    }
                }
                Thread.sleep(sleepTimeMS);
            } catch (LockObtainFailedException loe) {
            // obtain failed
            }
            if (i % 500 == 0) {
                System.out.println((i * 100. / count) + "% done.");
            }
            Thread.sleep(sleepTimeMS);
        }
    }
    System.out.println("Finished " + count + " tries.");
}

Example 7 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class PrintTaxonomyStats method main.

/** Command-line tool. */
@SuppressForbidden(reason = "System.out required: command line tool")
public static void main(String[] args) throws IOException {
    boolean printTree = false;
    String path = null;
    for (int i = 0; i < args.length; i++) {
        if (args[i].equals("-printTree")) {
            printTree = true;
        } else {
            path = args[i];
        }
    }
    if (args.length != (printTree ? 2 : 1)) {
        System.out.println("\nUsage: java -classpath ... org.apache.lucene.facet.util.PrintTaxonomyStats [-printTree] /path/to/taxononmy/index\n");
        System.exit(1);
    }
    Directory dir = FSDirectory.open(Paths.get(path));
    TaxonomyReader r = new DirectoryTaxonomyReader(dir);
    printStats(r, System.out, printTree);
    r.close();
    dir.close();
}

Example 8 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class TestStressNRTReplication method startNode.

/** Launches a child "server" (separate JVM), which is either primary or replica node */
@SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")
NodeProcess startNode(final int id, Path indexPath, boolean isPrimary, long forcePrimaryVersion) throws IOException {
    nodeTimeStamps[id] = System.nanoTime();
    List<String> cmd = new ArrayList<>();
    NodeProcess curPrimary = primary;
    cmd.add(System.getProperty("java.home") + System.getProperty("file.separator") + "bin" + System.getProperty("file.separator") + "java");
    cmd.add("-Xmx512m");
    if (curPrimary != null) {
        cmd.add("-Dtests.nrtreplication.primaryTCPPort=" + curPrimary.tcpPort);
    } else if (isPrimary == false) {
        // We cannot start a replica when there is no primary:
        return null;
    }
    // This is very costly (takes more time to check than it did to index); we do this ourselves in the end instead of each time a replica
    // is restarted:
    // cmd.add("-Dtests.nrtreplication.checkonclose=true");
    cmd.add("-Dtests.nrtreplication.node=true");
    cmd.add("-Dtests.nrtreplication.nodeid=" + id);
    cmd.add("-Dtests.nrtreplication.startNS=" + Node.globalStartNS);
    cmd.add("-Dtests.nrtreplication.indexpath=" + indexPath);
    if (isPrimary) {
        cmd.add("-Dtests.nrtreplication.isPrimary=true");
        cmd.add("-Dtests.nrtreplication.forcePrimaryVersion=" + forcePrimaryVersion);
        if (DO_CRASH_PRIMARY) {
            cmd.add("-Dtests.nrtreplication.doRandomCrash=true");
        }
        if (DO_CLOSE_PRIMARY) {
            cmd.add("-Dtests.nrtreplication.doRandomClose=true");
        }
    } else {
        if (DO_CRASH_REPLICA) {
            cmd.add("-Dtests.nrtreplication.doRandomCrash=true");
        }
        if (DO_CLOSE_REPLICA) {
            cmd.add("-Dtests.nrtreplication.doRandomClose=true");
        }
    }
    if (DO_BIT_FLIPS_DURING_COPY) {
        cmd.add("-Dtests.nrtreplication.doFlipBitsDuringCopy=true");
    }
    long myPrimaryGen = primaryGen;
    cmd.add("-Dtests.nrtreplication.primaryGen=" + myPrimaryGen);
    // Mixin our own counter because this is called from a fresh thread which means the seed otherwise isn't changing each time we spawn a
    // new node:
    long seed = random().nextLong() * nodeStartCounter.incrementAndGet();
    cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed));
    cmd.add("-ea");
    cmd.add("-cp");
    cmd.add(System.getProperty("java.class.path"));
    cmd.add("org.junit.runner.JUnitCore");
    cmd.add(getClass().getName().replace(getClass().getSimpleName(), "SimpleServer"));
    Writer childLog;
    if (SEPARATE_CHILD_OUTPUT) {
        Path childOut = childTempDir.resolve(id + ".log");
        message("logging to " + childOut);
        childLog = Files.newBufferedWriter(childOut, StandardCharsets.UTF_8, StandardOpenOption.APPEND, StandardOpenOption.CREATE);
        childLog.write("\n\nSTART NEW CHILD:\n");
    } else {
        childLog = null;
    }
    //message("child process command: " + cmd);
    ProcessBuilder pb = new ProcessBuilder(cmd);
    pb.redirectErrorStream(true);
    // Important, so that the scary looking hs_err_<pid>.log appear under our test temp dir:
    pb.directory(childTempDir.toFile());
    Process p = pb.start();
    BufferedReader r;
    try {
        r = new BufferedReader(new InputStreamReader(p.getInputStream(), IOUtils.UTF_8));
    } catch (UnsupportedEncodingException uee) {
        throw new RuntimeException(uee);
    }
    int tcpPort = -1;
    long initCommitVersion = -1;
    long initInfosVersion = -1;
    Pattern logTimeStart = Pattern.compile("^[0-9\\.]+s .*");
    boolean willCrash = false;
    while (true) {
        String l = r.readLine();
        if (l == null) {
            message("top: node=" + id + " failed to start");
            try {
                p.waitFor();
            } catch (InterruptedException ie) {
                throw new RuntimeException(ie);
            }
            message("exit value=" + p.exitValue());
            if (p.exitValue() == 0) {
                message("zero exit status; assuming failed to remove segments_N; skipping");
                return null;
            }
            // Hackity hack, in case primary crashed/closed and we haven't noticed (reaped the process) yet:
            if (isPrimary == false) {
                for (int i = 0; i < 100; i++) {
                    NodeProcess primary2 = primary;
                    if (primaryGen != myPrimaryGen || primary2 == null || primary2.nodeIsClosing.get()) {
                        // OK: primary crashed while we were trying to start, so it's expected/allowed that we could not start the replica:
                        message("primary crashed/closed while replica R" + id + " tried to start; skipping");
                        return null;
                    } else {
                        try {
                            Thread.sleep(10);
                        } catch (InterruptedException ie) {
                            throw new ThreadInterruptedException(ie);
                        }
                    }
                }
            }
            // Should fail the test:
            message("top: now fail test replica R" + id + " failed to start");
            failed.set(true);
            throw new RuntimeException("replica R" + id + " failed to start");
        }
        if (childLog != null) {
            childLog.write(l);
            childLog.write("\n");
            childLog.flush();
        } else if (logTimeStart.matcher(l).matches()) {
            // Already a well-formed log output:
            System.out.println(l);
        } else {
            message(l);
        }
        if (l.startsWith("PORT: ")) {
            tcpPort = Integer.parseInt(l.substring(6).trim());
        } else if (l.startsWith("COMMIT VERSION: ")) {
            initCommitVersion = Integer.parseInt(l.substring(16).trim());
        } else if (l.startsWith("INFOS VERSION: ")) {
            initInfosVersion = Integer.parseInt(l.substring(15).trim());
        } else if (l.contains("will crash after")) {
            willCrash = true;
        } else if (l.startsWith("NODE STARTED")) {
            break;
        }
    }
    final boolean finalWillCrash = willCrash;
    final AtomicBoolean nodeIsClosing = new AtomicBoolean();
    // Baby sits the child process, pulling its stdout and printing to our stdout, calling nodeClosed once it exits:
    Thread pumper = ThreadPumper.start(new Runnable() {

        @Override
        public void run() {
            message("now wait for process " + p);
            try {
                p.waitFor();
            } catch (Throwable t) {
                throw new RuntimeException(t);
            }
            message("done wait for process " + p);
            int exitValue = p.exitValue();
            message("exit value=" + exitValue + " willCrash=" + finalWillCrash);
            if (childLog != null) {
                try {
                    childLog.write("process done; exitValue=" + exitValue + "\n");
                    childLog.close();
                } catch (IOException ioe) {
                    throw new RuntimeException(ioe);
                }
            }
            if (exitValue != 0 && finalWillCrash == false && crashingNodes.remove(id) == false) {
                // should fail test
                failed.set(true);
                if (childLog != null) {
                    throw new RuntimeException("node " + id + " process had unexpected non-zero exit status=" + exitValue + "; see " + childLog + " for details");
                } else {
                    throw new RuntimeException("node " + id + " process had unexpected non-zero exit status=" + exitValue);
                }
            }
            nodeClosed(id);
        }
    }, r, System.out, childLog, nodeIsClosing);
    pumper.setName("pump" + id);
    message("top: node=" + id + " started at tcpPort=" + tcpPort + " initCommitVersion=" + initCommitVersion + " initInfosVersion=" + initInfosVersion);
    return new NodeProcess(p, id, tcpPort, pumper, isPrimary, initCommitVersion, initInfosVersion, nodeIsClosing);
}

Example 9 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class TestNRTReplication method startNode.

/** Launches a child "server" (separate JVM), which is either primary or replica node */
@SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")
private NodeProcess startNode(int primaryTCPPort, final int id, Path indexPath, long forcePrimaryVersion, boolean willCrash) throws IOException {
    List<String> cmd = new ArrayList<>();
    cmd.add(System.getProperty("java.home") + System.getProperty("file.separator") + "bin" + System.getProperty("file.separator") + "java");
    cmd.add("-Xmx512m");
    long myPrimaryGen;
    if (primaryTCPPort != -1) {
        // I am a replica
        cmd.add("-Dtests.nrtreplication.primaryTCPPort=" + primaryTCPPort);
        myPrimaryGen = lastPrimaryGen;
    } else {
        myPrimaryGen = nextPrimaryGen++;
        lastPrimaryGen = myPrimaryGen;
    }
    cmd.add("-Dtests.nrtreplication.primaryGen=" + myPrimaryGen);
    cmd.add("-Dtests.nrtreplication.closeorcrash=false");
    cmd.add("-Dtests.nrtreplication.node=true");
    cmd.add("-Dtests.nrtreplication.nodeid=" + id);
    cmd.add("-Dtests.nrtreplication.startNS=" + Node.globalStartNS);
    cmd.add("-Dtests.nrtreplication.indexpath=" + indexPath);
    cmd.add("-Dtests.nrtreplication.checkonclose=true");
    if (primaryTCPPort == -1) {
        // We are the primary node
        cmd.add("-Dtests.nrtreplication.isPrimary=true");
        cmd.add("-Dtests.nrtreplication.forcePrimaryVersion=" + forcePrimaryVersion);
    }
    // Mixin our own counter because this is called from a fresh thread which means the seed otherwise isn't changing each time we spawn a
    // new node:
    long seed = random().nextLong() * nodeStartCounter.incrementAndGet();
    cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed));
    cmd.add("-ea");
    cmd.add("-cp");
    cmd.add(System.getProperty("java.class.path"));
    cmd.add("org.junit.runner.JUnitCore");
    cmd.add(getClass().getName().replace(getClass().getSimpleName(), "SimpleServer"));
    message("child process command: " + cmd);
    ProcessBuilder pb = new ProcessBuilder(cmd);
    pb.redirectErrorStream(true);
    // Important, so that the scary looking hs_err_<pid>.log appear under our test temp dir:
    pb.directory(childTempDir.toFile());
    Process p = pb.start();
    BufferedReader r;
    try {
        r = new BufferedReader(new InputStreamReader(p.getInputStream(), IOUtils.UTF_8));
    } catch (UnsupportedEncodingException uee) {
        throw new RuntimeException(uee);
    }
    int tcpPort = -1;
    long initCommitVersion = -1;
    long initInfosVersion = -1;
    Pattern logTimeStart = Pattern.compile("^[0-9\\.]+s .*");
    boolean sawExistingSegmentsFile = false;
    while (true) {
        String l = r.readLine();
        if (l == null) {
            message("top: node=" + id + " failed to start");
            try {
                p.waitFor();
            } catch (InterruptedException ie) {
                throw new RuntimeException(ie);
            }
            message("exit value=" + p.exitValue());
            message("top: now fail test replica R" + id + " failed to start");
            throw new RuntimeException("replica R" + id + " failed to start");
        }
        if (logTimeStart.matcher(l).matches()) {
            // Already a well-formed log output:
            System.out.println(l);
        } else {
            message(l);
        }
        if (l.startsWith("PORT: ")) {
            tcpPort = Integer.parseInt(l.substring(6).trim());
        } else if (l.startsWith("COMMIT VERSION: ")) {
            initCommitVersion = Integer.parseInt(l.substring(16).trim());
        } else if (l.startsWith("INFOS VERSION: ")) {
            initInfosVersion = Integer.parseInt(l.substring(15).trim());
        } else if (l.contains("will crash after")) {
            willCrash = true;
        } else if (l.startsWith("NODE STARTED")) {
            break;
        } else if (l.contains("replica cannot start: existing segments file=")) {
            sawExistingSegmentsFile = true;
        }
    }
    final boolean finalWillCrash = willCrash;
    // Baby sits the child process, pulling its stdout and printing to our stdout:
    AtomicBoolean nodeClosing = new AtomicBoolean();
    Thread pumper = ThreadPumper.start(new Runnable() {

        @Override
        public void run() {
            message("now wait for process " + p);
            try {
                p.waitFor();
            } catch (Throwable t) {
                throw new RuntimeException(t);
            }
            message("done wait for process " + p);
            int exitValue = p.exitValue();
            message("exit value=" + exitValue + " willCrash=" + finalWillCrash);
            if (exitValue != 0 && finalWillCrash == false) {
                // should fail test
                throw new RuntimeException("node " + id + " process had unexpected non-zero exit status=" + exitValue);
            }
        }
    }, r, System.out, null, nodeClosing);
    pumper.setName("pump" + id);
    message("top: node=" + id + " started at tcpPort=" + tcpPort + " initCommitVersion=" + initCommitVersion + " initInfosVersion=" + initInfosVersion);
    return new NodeProcess(p, id, tcpPort, pumper, primaryTCPPort == -1, initCommitVersion, initInfosVersion, nodeClosing);
}

Example 10 with SuppressForbidden

use of org.apache.lucene.util.SuppressForbidden in project lucene-solr by apache.

the class IndexSplitter method listSegments.

@SuppressForbidden(reason = "System.out required: command line tool")
public void listSegments() throws IOException {
    DecimalFormat formatter = new DecimalFormat("###,###.###", DecimalFormatSymbols.getInstance(Locale.ROOT));
    for (int x = 0; x < infos.size(); x++) {
        SegmentCommitInfo info = infos.info(x);
        String sizeStr = formatter.format(info.sizeInBytes());
        System.out.println(info.info.name + " " + sizeStr);
    }
}