Examples with Callable java.util.concurrent.Callable used on opensource projects

Example 91 with Callable

use of java.util.concurrent.Callable in project lucene-solr by apache.

the class IndexSortedFacetCollector method getFacetCounts.

NamedList<Integer> getFacetCounts(Executor executor) throws IOException {
    CompletionService<SegFacet> completionService = new ExecutorCompletionService<>(executor);
    // reuse the translation logic to go from top level set to per-segment set
    baseSet = docs.getTopFilter();
    final List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves();
    // The list of pending tasks that aren't immediately submitted
    // TODO: Is there a completion service, or a delegating executor that can
    // limit the number of concurrent tasks submitted to a bigger executor?
    LinkedList<Callable<SegFacet>> pending = new LinkedList<>();
    int threads = nThreads <= 0 ? Integer.MAX_VALUE : nThreads;
    for (final LeafReaderContext leave : leaves) {
        final SegFacet segFacet = new SegFacet(leave);
        Callable<SegFacet> task = () -> {
            segFacet.countTerms();
            return segFacet;
        };
        if (--threads >= 0) {
            completionService.submit(task);
        } else {
            pending.add(task);
        }
    }
    // now merge the per-segment results
    PriorityQueue<SegFacet> queue = new PriorityQueue<SegFacet>(leaves.size()) {

        @Override
        protected boolean lessThan(SegFacet a, SegFacet b) {
            return a.tempBR.compareTo(b.tempBR) < 0;
        }
    };
    boolean hasMissingCount = false;
    int missingCount = 0;
    for (int i = 0, c = leaves.size(); i < c; i++) {
        SegFacet seg = null;
        try {
            Future<SegFacet> future = completionService.take();
            seg = future.get();
            if (!pending.isEmpty()) {
                completionService.submit(pending.removeFirst());
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            if (cause instanceof RuntimeException) {
                throw (RuntimeException) cause;
            } else {
                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in per-segment faceting on field: " + fieldName, cause);
            }
        }
        if (seg.startTermIndex < seg.endTermIndex) {
            if (seg.startTermIndex == -1) {
                hasMissingCount = true;
                missingCount += seg.counts[0];
                seg.pos = 0;
            } else {
                seg.pos = seg.startTermIndex;
            }
            if (seg.pos < seg.endTermIndex && (mincount < 1 || seg.hasAnyCount)) {
                seg.tenum = seg.si.termsEnum();
                seg.tenum.seekExact(seg.pos);
                seg.tempBR = seg.tenum.term();
                queue.add(seg);
            }
        }
    }
    FacetCollector collector;
    if (sort.equals(FacetParams.FACET_SORT_COUNT) || sort.equals(FacetParams.FACET_SORT_COUNT_LEGACY)) {
        collector = new CountSortedFacetCollector(offset, limit, mincount);
    } else {
        collector = new IndexSortedFacetCollector(offset, limit, mincount);
    }
    BytesRefBuilder val = new BytesRefBuilder();
    while (queue.size() > 0) {
        SegFacet seg = queue.top();
        boolean collect = termFilter == null || termFilter.test(seg.tempBR);
        // may be shared across calls.
        if (collect) {
            val.copyBytes(seg.tempBR);
        }
        int count = 0;
        do {
            if (collect) {
                count += seg.counts[seg.pos - seg.startTermIndex];
            }
            // if mincount>0 then seg.pos++ can skip ahead to the next non-zero entry.
            do {
                ++seg.pos;
            } while (//stop incrementing before we run off the end
            (seg.pos < seg.endTermIndex) && //move term enum forward with position -- dont care about value 
            (seg.tenum.next() != null || true) && //only skip ahead if mincount > 0
            (mincount > 0) && //check zero count
            (seg.counts[seg.pos - seg.startTermIndex] == 0));
            if (seg.pos >= seg.endTermIndex) {
                queue.pop();
                seg = queue.top();
            } else {
                seg.tempBR = seg.tenum.term();
                seg = queue.updateTop();
            }
        } while (seg != null && val.get().compareTo(seg.tempBR) == 0);
        if (collect) {
            boolean stop = collector.collect(val.get(), count);
            if (stop)
                break;
        }
    }
    NamedList<Integer> res = collector.getFacetCounts();
    // convert labels to readable form    
    FieldType ft = searcher.getSchema().getFieldType(fieldName);
    int sz = res.size();
    for (int i = 0; i < sz; i++) {
        res.setName(i, ft.indexedToReadable(res.getName(i)));
    }
    if (missing) {
        if (!hasMissingCount) {
            missingCount = SimpleFacets.getFieldMissingCount(searcher, docs, fieldName);
        }
        res.add(null, missingCount);
    }
    return res;
}

Example 92 with Callable

use of java.util.concurrent.Callable in project lucene-solr by apache.

the class SubQueryAugmenter method transform.

@Override
public void transform(SolrDocument doc, int docid, float score) {
    final SolrParams docWithDeprefixed = SolrParams.wrapDefaults(new DocRowParams(doc, prefix, separator), baseSubParams);
    try {
        Callable<QueryResponse> subQuery = new Callable<QueryResponse>() {

            @Override
            public QueryResponse call() throws Exception {
                try {
                    return new QueryResponse(server.request(new QueryRequest(docWithDeprefixed), coreName), server);
                } finally {
                }
            }
        };
        QueryResponse response = SolrRequestInfoSuspender.doInSuspension(subQuery);
        final SolrDocumentList docList = (SolrDocumentList) response.getResults();
        doc.setField(getName(), new Result(docList));
    } catch (Exception e) {
        String docString = doc.toString();
        throw new SolrException(ErrorCode.BAD_REQUEST, "while invoking " + name + ":[subquery" + (coreName != null ? "fromIndex=" + coreName : "") + "] on doc=" + docString.substring(0, Math.min(100, docString.length())), e.getCause());
    } finally {
    }
}

Example 93 with Callable

use of java.util.concurrent.Callable in project lucene-solr by apache.

the class IndexAndTaxonomyReplicationClientTest method testConsistencyOnExceptions.

/*
   * This test verifies that the client and handler do not end up in a corrupt
   * index if exceptions are thrown at any point during replication. Either when
   * a client copies files from the server to the temporary space, or when the
   * handler copies them to the index directory.
   */
@Test
public void testConsistencyOnExceptions() throws Exception {
    // so the handler's index isn't empty
    replicator.publish(createRevision(1));
    client.updateNow();
    client.close();
    callback.close();
    // wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
    final SourceDirectoryFactory in = sourceDirFactory;
    final AtomicInteger failures = new AtomicInteger(atLeast(10));
    sourceDirFactory = new SourceDirectoryFactory() {

        private long clientMaxSize = 100, handlerIndexMaxSize = 100, handlerTaxoMaxSize = 100;

        private double clientExRate = 1.0, handlerIndexExRate = 1.0, handlerTaxoExRate = 1.0;

        @Override
        public void cleanupSession(String sessionID) throws IOException {
            in.cleanupSession(sessionID);
        }

        @SuppressWarnings("synthetic-access")
        @Override
        public Directory getDirectory(String sessionID, String source) throws IOException {
            Directory dir = in.getDirectory(sessionID, source);
            if (random().nextBoolean() && failures.get() > 0) {
                // client should fail, return wrapped dir
                MockDirectoryWrapper mdw = new MockDirectoryWrapper(random(), dir);
                mdw.setRandomIOExceptionRateOnOpen(clientExRate);
                mdw.setMaxSizeInBytes(clientMaxSize);
                mdw.setRandomIOExceptionRate(clientExRate);
                mdw.setCheckIndexOnClose(false);
                clientMaxSize *= 2;
                clientExRate /= 2;
                return mdw;
            }
            if (failures.get() > 0 && random().nextBoolean()) {
                // handler should fail
                if (random().nextBoolean()) {
                    // index dir fail
                    handlerIndexDir.setMaxSizeInBytes(handlerIndexMaxSize);
                    handlerIndexDir.setRandomIOExceptionRate(handlerIndexExRate);
                    handlerIndexDir.setRandomIOExceptionRateOnOpen(handlerIndexExRate);
                    handlerIndexMaxSize *= 2;
                    handlerIndexExRate /= 2;
                } else {
                    // taxo dir fail
                    handlerTaxoDir.setMaxSizeInBytes(handlerTaxoMaxSize);
                    handlerTaxoDir.setRandomIOExceptionRate(handlerTaxoExRate);
                    handlerTaxoDir.setRandomIOExceptionRateOnOpen(handlerTaxoExRate);
                    handlerTaxoDir.setCheckIndexOnClose(false);
                    handlerTaxoMaxSize *= 2;
                    handlerTaxoExRate /= 2;
                }
            } else {
                // disable all errors
                handlerIndexDir.setMaxSizeInBytes(0);
                handlerIndexDir.setRandomIOExceptionRate(0.0);
                handlerIndexDir.setRandomIOExceptionRateOnOpen(0.0);
                handlerTaxoDir.setMaxSizeInBytes(0);
                handlerTaxoDir.setRandomIOExceptionRate(0.0);
                handlerTaxoDir.setRandomIOExceptionRateOnOpen(0.0);
            }
            return dir;
        }
    };
    handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            if (random().nextDouble() < 0.2 && failures.get() > 0) {
                throw new RuntimeException("random exception from callback");
            }
            return null;
        }
    });
    final AtomicBoolean failed = new AtomicBoolean();
    // wrap handleUpdateException so we can act on the thrown exception
    client = new ReplicationClient(replicator, handler, sourceDirFactory) {

        @SuppressWarnings("synthetic-access")
        @Override
        protected void handleUpdateException(Throwable t) {
            if (t instanceof IOException) {
                try {
                    if (VERBOSE) {
                        System.out.println("hit exception during update: " + t);
                        t.printStackTrace(System.out);
                    }
                    // test that the index can be read and also some basic statistics
                    DirectoryReader reader = DirectoryReader.open(handlerIndexDir.getDelegate());
                    try {
                        int numDocs = reader.numDocs();
                        int version = Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
                        assertEquals(numDocs, version);
                    } finally {
                        reader.close();
                    }
                    // verify index is fully consistent
                    TestUtil.checkIndex(handlerIndexDir.getDelegate());
                    // verify taxonomy index is fully consistent (since we only add one
                    // category to all documents, there's nothing much more to validate.
                    ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
                    CheckIndex.Status indexStatus = null;
                    try (CheckIndex checker = new CheckIndex(handlerTaxoDir.getDelegate())) {
                        checker.setFailFast(true);
                        checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
                        try {
                            indexStatus = checker.checkIndex(null);
                        } catch (IOException | RuntimeException ioe) {
                        // ok: we fallback below
                        }
                    }
                } catch (IOException e) {
                    failed.set(true);
                    throw new RuntimeException(e);
                } catch (RuntimeException e) {
                    failed.set(true);
                    throw e;
                } finally {
                    // count-down number of failures
                    failures.decrementAndGet();
                    assert failures.get() >= 0 : "handler failed too many times: " + failures.get();
                    if (VERBOSE) {
                        if (failures.get() == 0) {
                            System.out.println("no more failures expected");
                        } else {
                            System.out.println("num failures left: " + failures.get());
                        }
                    }
                }
            } else {
                failed.set(true);
                if (t instanceof RuntimeException) {
                    throw (RuntimeException) t;
                }
                throw new RuntimeException(t);
            }
        }
    };
    client.startUpdateThread(10, "indexAndTaxo");
    final Directory baseHandlerIndexDir = handlerIndexDir.getDelegate();
    int numRevisions = atLeast(20) + 2;
    for (int i = 2; i < numRevisions && failed.get() == false; i++) {
        replicator.publish(createRevision(i));
        assertHandlerRevision(i, baseHandlerIndexDir);
    }
    // disable errors -- maybe randomness didn't exhaust all allowed failures,
    // and we don't want e.g. CheckIndex to hit false errors. 
    handlerIndexDir.setMaxSizeInBytes(0);
    handlerIndexDir.setRandomIOExceptionRate(0.0);
    handlerIndexDir.setRandomIOExceptionRateOnOpen(0.0);
    handlerTaxoDir.setMaxSizeInBytes(0);
    handlerTaxoDir.setRandomIOExceptionRate(0.0);
    handlerTaxoDir.setRandomIOExceptionRateOnOpen(0.0);
}

Example 94 with Callable

use of java.util.concurrent.Callable in project lucene-solr by apache.

the class IndexReplicationClientTest method testConsistencyOnExceptions.

/*
   * This test verifies that the client and handler do not end up in a corrupt
   * index if exceptions are thrown at any point during replication. Either when
   * a client copies files from the server to the temporary space, or when the
   * handler copies them to the index directory.
   */
@Test
public void testConsistencyOnExceptions() throws Exception {
    // so the handler's index isn't empty
    replicator.publish(createRevision(1));
    client.updateNow();
    client.close();
    callback.close();
    // wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
    final SourceDirectoryFactory in = sourceDirFactory;
    final AtomicInteger failures = new AtomicInteger(atLeast(10));
    sourceDirFactory = new SourceDirectoryFactory() {

        private long clientMaxSize = 100, handlerMaxSize = 100;

        private double clientExRate = 1.0, handlerExRate = 1.0;

        @Override
        public void cleanupSession(String sessionID) throws IOException {
            in.cleanupSession(sessionID);
        }

        @SuppressWarnings("synthetic-access")
        @Override
        public Directory getDirectory(String sessionID, String source) throws IOException {
            Directory dir = in.getDirectory(sessionID, source);
            if (random().nextBoolean() && failures.get() > 0) {
                // client should fail, return wrapped dir
                MockDirectoryWrapper mdw = new MockDirectoryWrapper(random(), dir);
                mdw.setRandomIOExceptionRateOnOpen(clientExRate);
                mdw.setMaxSizeInBytes(clientMaxSize);
                mdw.setRandomIOExceptionRate(clientExRate);
                mdw.setCheckIndexOnClose(false);
                clientMaxSize *= 2;
                clientExRate /= 2;
                return mdw;
            }
            if (failures.get() > 0 && random().nextBoolean()) {
                // handler should fail
                handlerDir.setMaxSizeInBytes(handlerMaxSize);
                handlerDir.setRandomIOExceptionRateOnOpen(handlerExRate);
                handlerDir.setRandomIOExceptionRate(handlerExRate);
                handlerMaxSize *= 2;
                handlerExRate /= 2;
            } else {
                // disable errors
                handlerDir.setMaxSizeInBytes(0);
                handlerDir.setRandomIOExceptionRate(0.0);
                handlerDir.setRandomIOExceptionRateOnOpen(0.0);
            }
            return dir;
        }
    };
    handler = new IndexReplicationHandler(handlerDir, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            if (random().nextDouble() < 0.2 && failures.get() > 0) {
                throw new RuntimeException("random exception from callback");
            }
            return null;
        }
    });
    // wrap handleUpdateException so we can act on the thrown exception
    client = new ReplicationClient(replicator, handler, sourceDirFactory) {

        @SuppressWarnings("synthetic-access")
        @Override
        protected void handleUpdateException(Throwable t) {
            if (t instanceof IOException) {
                if (VERBOSE) {
                    System.out.println("hit exception during update: " + t);
                    t.printStackTrace(System.out);
                }
                try {
                    // test that the index can be read and also some basic statistics
                    DirectoryReader reader = DirectoryReader.open(handlerDir.getDelegate());
                    try {
                        int numDocs = reader.numDocs();
                        int version = Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
                        assertEquals(numDocs, version);
                    } finally {
                        reader.close();
                    }
                    // verify index consistency
                    TestUtil.checkIndex(handlerDir.getDelegate());
                } catch (IOException e) {
                    // exceptions here are bad, don't ignore them
                    throw new RuntimeException(e);
                } finally {
                    // count-down number of failures
                    failures.decrementAndGet();
                    assert failures.get() >= 0 : "handler failed too many times: " + failures.get();
                    if (VERBOSE) {
                        if (failures.get() == 0) {
                            System.out.println("no more failures expected");
                        } else {
                            System.out.println("num failures left: " + failures.get());
                        }
                    }
                }
            } else {
                if (t instanceof RuntimeException)
                    throw (RuntimeException) t;
                throw new RuntimeException(t);
            }
        }
    };
    client.startUpdateThread(10, "index");
    final Directory baseHandlerDir = handlerDir.getDelegate();
    int numRevisions = atLeast(20);
    for (int i = 2; i < numRevisions; i++) {
        replicator.publish(createRevision(i));
        assertHandlerRevision(i, baseHandlerDir);
    }
    // disable errors -- maybe randomness didn't exhaust all allowed failures,
    // and we don't want e.g. CheckIndex to hit false errors. 
    handlerDir.setMaxSizeInBytes(0);
    handlerDir.setRandomIOExceptionRate(0.0);
    handlerDir.setRandomIOExceptionRateOnOpen(0.0);
}

Example 95 with Callable

use of java.util.concurrent.Callable in project jackrabbit-oak by apache.

the class ConcurrentTest method testCacheAccessInLoaderDeadlock.

@Test
public void testCacheAccessInLoaderDeadlock() throws Exception {
    final Random r = new Random(1);
    final CacheLIRS<Integer, Integer> cache = new CacheLIRS.Builder<Integer, Integer>().maximumWeight(100).averageWeight(10).build();
    final Exception[] ex = new Exception[1];
    final int entryCount = 10;
    int size = 3;
    Thread[] threads = new Thread[size];
    final AtomicBoolean stop = new AtomicBoolean();
    for (int i = 0; i < size; i++) {
        Thread t = new Thread() {

            @Override
            public void run() {
                Callable<Integer> callable = new Callable<Integer>() {

                    @Override
                    public Integer call() throws ExecutionException {
                        if (r.nextBoolean()) {
                            cache.get(r.nextInt(entryCount), this);
                        } else {
                            cache.get(r.nextInt(entryCount));
                        }
                        return 1;
                    }
                };
                while (!stop.get()) {
                    Integer key = r.nextInt(entryCount);
                    try {
                        cache.get(key, callable);
                    } catch (Exception e) {
                        ex[0] = e;
                    }
                    cache.remove(key);
                }
            }
        };
        t.start();
        threads[i] = t;
    }
    // test for 100 ms
    Thread.sleep(100);
    stop.set(true);
    for (Thread t : threads) {
        t.join(1000);
        // but report a failure (what else could we do?)
        if (t.isAlive()) {
            assertFalse("Deadlock detected!", t.isAlive());
        }
    }
    if (ex[0] != null) {
        throw ex[0];
    }
}