Examples with Callable java.util.concurrent.Callable used on opensource projects

Example 51 with Callable

use of java.util.concurrent.Callable in project nutz by nutzam.

the class Scans method addConcurrency.

public boolean addConcurrency(String[] _jars) {
    ExecutorService es = Executors.newFixedThreadPool(8);
    List<Future<ResourceLocation>> tmps = new ArrayList<Future<ResourceLocation>>(_jars.length);
    for (int i = 0; i < _jars.length; i++) {
        final String path = _jars[i];
        if (path == null || !path.toLowerCase().endsWith(".jar"))
            continue;
        tmps.add(es.submit(new Callable<ResourceLocation>() {

            public ResourceLocation call() {
                return ResourceLocation.jar(path);
            }
        }));
    }
    es.shutdown();
    try {
        es.awaitTermination(5, TimeUnit.MINUTES);
        for (Future<ResourceLocation> f : tmps) {
            try {
                if (f.get() != null)
                    addResourceLocation(f.get());
            } catch (ExecutionException e) {
            }
        }
        return true;
    } catch (InterruptedException e) {
        return false;
    }
}

Example 52 with Callable

use of java.util.concurrent.Callable in project neo4j by neo4j.

the class ConcurrentAccessIT method newWorker.

private Callable<Void> newWorker(final int iterationsToRun) throws Exception {
    return new Callable<Void>() {

        private final byte[] init = chunk(InitMessage.init("TestClient", emptyMap()));

        private final byte[] createAndRollback = chunk(run("BEGIN"), pullAll(), run("CREATE (n)"), pullAll(), run("ROLLBACK"), pullAll());

        private final byte[] matchAll = chunk(run("MATCH (n) RETURN n"), pullAll());

        @Override
        public Void call() throws Exception {
            // Connect
            TransportConnection client = cf.newInstance();
            client.connect(address).send(acceptedVersions(1, 0, 0, 0));
            assertThat(client, eventuallyReceives(new byte[] { 0, 0, 0, 1 }));
            init(client);
            for (int i = 0; i < iterationsToRun; i++) {
                createAndRollback(client);
            }
            return null;
        }

        private void init(TransportConnection client) throws Exception {
            client.send(init);
            assertThat(client, eventuallyReceives(msgSuccess()));
        }

        private void createAndRollback(TransportConnection client) throws Exception {
            client.send(createAndRollback);
            assertThat(client, eventuallyReceives(msgSuccess(allOf(hasEntry(is("fields"), equalTo(emptyList())), hasKey("result_available_after"))), msgSuccess(), msgSuccess(allOf(hasEntry(is("fields"), equalTo(emptyList())), hasKey("result_available_after"))), msgSuccess(), msgSuccess(allOf(hasEntry(is("fields"), equalTo(emptyList())), hasKey("result_available_after"))), msgSuccess()));
            // Verify no visible data
            client.send(matchAll);
            assertThat(client, eventuallyReceives(msgSuccess(allOf(hasEntry(is("fields"), equalTo(singletonList("n"))), hasKey("result_available_after"))), msgSuccess()));
        }
    };
}

Example 53 with Callable

use of java.util.concurrent.Callable in project Xponents by OpenSextant.

the class CloserThread method load.

//-------------------------------------------------------------------
// Initialization / Cleanup
//-------------------------------------------------------------------
/**
   * Load the cores defined for this CoreContainer
   */
public void load() {
    log.info("Loading cores into CoreContainer [instanceDir={}]", loader.getInstanceDir());
    // add the sharedLib to the shared resource loader before initializing cfg based plugins
    String libDir = cfg.getSharedLibDirectory();
    if (libDir != null) {
        File f = FileUtils.resolvePath(new File(solrHome), libDir);
        log.info("loading shared library: " + f.getAbsolutePath());
        loader.addToClassLoader(libDir, null, false);
        loader.reloadLuceneSPI();
    }
    shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
    updateShardHandler = new UpdateShardHandler(cfg);
    solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);
    logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
    hostName = cfg.getHost();
    log.info("Host Name: " + hostName);
    zkSys.initZooKeeper(this, solrHome, cfg);
    collectionsHandler = createHandler(cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
    infoHandler = createHandler(cfg.getInfoHandlerClass(), InfoHandler.class);
    coreAdminHandler = createHandler(cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
    coreConfigService = cfg.createCoreConfigService(loader, zkSys.getZkController());
    containerProperties = cfg.getSolrProperties("solr");
    // setup executor to load cores in parallel
    // do not limit the size of the executor in zk mode since cores may try and wait for each other.
    ExecutorService coreLoadExecutor = Executors.newFixedThreadPool((zkSys.getZkController() == null ? cfg.getCoreLoadThreadCount() : Integer.MAX_VALUE), new DefaultSolrThreadFactory("coreLoadExecutor"));
    // OpenSextant
    List<Future<SolrCore>> startupResults = Collections.emptyList();
    try {
        List<CoreDescriptor> cds = coresLocator.discover(this);
        checkForDuplicateCoreNames(cds);
        List<Callable<SolrCore>> creators = new ArrayList<>();
        for (final CoreDescriptor cd : cds) {
            if (cd.isTransient() || !cd.isLoadOnStartup()) {
                solrCores.putDynamicDescriptor(cd.getName(), cd);
            }
            if (cd.isLoadOnStartup()) {
                creators.add(new Callable<SolrCore>() {

                    @Override
                    public SolrCore call() throws Exception {
                        if (zkSys.getZkController() != null) {
                            zkSys.getZkController().throwErrorIfReplicaReplaced(cd);
                        }
                        return create(cd, false);
                    }
                });
            }
        }
        try {
            // changed by OpenSextant
            startupResults = coreLoadExecutor.invokeAll(creators);
        } catch (InterruptedException e) {
            throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Interrupted while loading cores");
        }
        // Start the background thread
        backgroundCloser = new CloserThread(this, solrCores, cfg);
        backgroundCloser.start();
    } finally {
        ExecutorUtil.shutdownNowAndAwaitTermination(coreLoadExecutor);
        // OpenSextant custom
        for (Future<SolrCore> core : startupResults) {
            try {
                core.get();
                log.info("Successfully loaded a core.");
            } catch (InterruptedException e) {
            // ignore, we've been cancelled
            } catch (ExecutionException e) {
                log.error("Error starting solr core.", e);
            }
        }
    // OpenSextant custom
    }
    if (isZooKeeperAware()) {
        // register in zk in background threads
        Collection<SolrCore> cores = getCores();
        if (cores != null) {
            for (SolrCore core : cores) {
                try {
                    zkSys.registerInZk(core, true);
                } catch (Throwable t) {
                    SolrException.log(log, "Error registering SolrCore", t);
                }
            }
        }
        zkSys.getZkController().checkOverseerDesignate();
    }
}

Example 54 with Callable

use of java.util.concurrent.Callable in project opennms by OpenNMS.

the class NewtsFetchStrategy method getMeasurementsForResourceCallable.

private Callable<Collection<Row<Measurement>>> getMeasurementsForResourceCallable(final String newtsResourceId, final List<Source> listOfSources, final Optional<Timestamp> start, final Optional<Timestamp> end, final LateAggregationParams lag) {
    return new Callable<Collection<Row<Measurement>>>() {

        @Override
        public Collection<Row<Measurement>> call() throws Exception {
            ResultDescriptor resultDescriptor = new ResultDescriptor(lag.getInterval());
            for (Source source : listOfSources) {
                final String metricName = source.getAttribute();
                final String name = source.getLabel();
                final AggregationFunction fn = toAggregationFunction(source.getAggregation());
                resultDescriptor.datasource(name, metricName, lag.getHeartbeat(), fn);
                resultDescriptor.export(name);
            }
            LOG.debug("Querying Newts for resource id {} with result descriptor: {}", newtsResourceId, resultDescriptor);
            Results<Measurement> results = m_sampleRepository.select(m_context, new Resource(newtsResourceId), start, end, resultDescriptor, Optional.of(Duration.millis(lag.getStep())), limitConcurrentAggregationsCallback);
            Collection<Row<Measurement>> rows = results.getRows();
            LOG.debug("Found {} rows.", rows.size());
            return rows;
        }
    };
}

Example 55 with Callable

use of java.util.concurrent.Callable in project ProPPR by TeamCohen.

the class Diagnostic method main.

public static void main(String[] args) {
    StatusLogger status = new StatusLogger();
    try {
        int inputFiles = Configuration.USE_TRAIN;
        int outputFiles = 0;
        int constants = Configuration.USE_THREADS | Configuration.USE_THROTTLE;
        int modules = Configuration.USE_SRW;
        ModuleConfiguration c = new ModuleConfiguration(args, inputFiles, outputFiles, constants, modules);
        log.info(c.toString());
        String groundedFile = c.queryFile.getPath();
        log.info("Parsing " + groundedFile + "...");
        long start = System.currentTimeMillis();
        final ArrayLearningGraphBuilder b = new ArrayLearningGraphBuilder();
        final SRW srw = c.srw;
        final ParamVector<String, ?> params = srw.setupParams(new SimpleParamVector<String>(new ConcurrentHashMap<String, Double>(16, (float) 0.75, 24)));
        srw.setEpoch(1);
        srw.clearLoss();
        srw.fixedWeightRules().addExact("id(restart)");
        srw.fixedWeightRules().addExact("id(trueLoop)");
        srw.fixedWeightRules().addExact("id(trueLoopRestart)");
        /* DiagSrwES: */
        ArrayList<Future<PosNegRWExample>> parsed = new ArrayList<Future<PosNegRWExample>>();
        final ExecutorService trainerPool = Executors.newFixedThreadPool(c.nthreads > 1 ? c.nthreads / 2 : 1);
        final ExecutorService parserPool = Executors.newFixedThreadPool(c.nthreads > 1 ? c.nthreads / 2 : 1);
        int i = 1;
        for (String s : new ParsedFile(groundedFile)) {
            final int id = i++;
            final String in = s;
            parsed.add(parserPool.submit(new Callable<PosNegRWExample>() {

                @Override
                public PosNegRWExample call() throws Exception {
                    try {
                        //log.debug("Job start "+id);
                        //PosNegRWExample ret = parser.parse(in, b.copy());
                        log.debug("Parsing start " + id);
                        PosNegRWExample ret = new RWExampleParser().parse(in, b.copy(), srw);
                        log.debug("Parsing done " + id);
                        //log.debug("Job done "+id);
                        return ret;
                    } catch (IllegalArgumentException e) {
                        System.err.println("Problem with #" + id);
                        e.printStackTrace();
                    }
                    return null;
                }
            }));
        }
        parserPool.shutdown();
        i = 1;
        for (Future<PosNegRWExample> future : parsed) {
            final int id = i++;
            final Future<PosNegRWExample> in = future;
            trainerPool.submit(new Runnable() {

                @Override
                public void run() {
                    try {
                        PosNegRWExample ex = in.get();
                        log.debug("Training start " + id);
                        srw.trainOnExample(params, ex, status);
                        log.debug("Training done " + id);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    } catch (ExecutionException e) {
                        e.printStackTrace();
                    }
                }
            });
        }
        trainerPool.shutdown();
        try {
            parserPool.awaitTermination(7, TimeUnit.DAYS);
            trainerPool.awaitTermination(7, TimeUnit.DAYS);
        } catch (InterruptedException e) {
            log.error("Interrupted?", e);
        }
        /* /SrwES */
        /* SrwTtwop: 
			final ExecutorService parserPool = Executors.newFixedThreadPool(c.nthreads>1?c.nthreads/2:1);
			Multithreading<String,PosNegRWExample> m = new Multithreading<String,PosNegRWExample>(log);
			m.executeJob(c.nthreads/2, new ParsedFile(groundedFile), 
					new Transformer<String,PosNegRWExample>() {
						@Override
						public Callable<PosNegRWExample> transformer(final String in, final int id) {
							return new Callable<PosNegRWExample>() {
								@Override
								public PosNegRWExample call() throws Exception {
									try {
									//log.debug("Job start "+id);
									//PosNegRWExample ret = parser.parse(in, b.copy());
									log.debug("Parsing start "+id);
									PosNegRWExample ret = new GroundedExampleParser().parse(in, b.copy());
									log.debug("Parsing done "+id);
									//log.debug("Job done "+id);
									return ret;
									} catch (IllegalArgumentException e) {
										System.err.println("Problem with #"+id);
										e.printStackTrace();
									}
									return null;
								}};
						}}, new Cleanup<PosNegRWExample>() {
							@Override
							public Runnable cleanup(final Future<PosNegRWExample> in, final int id) {
								return new Runnable(){
									@Override
									public void run() {
										try {
											final PosNegRWExample ex = in.get();
											log.debug("Cleanup start "+id);
											trainerPool.submit(new Runnable() {
													@Override
													public void run(){
														log.debug("Training start "+id);
														srw.trainOnExample(params,ex);
														log.debug("Training done "+id);
													}
												});
										} catch (InterruptedException e) {
										    e.printStackTrace(); 
										} catch (ExecutionException e) {
										    e.printStackTrace();
										}
										log.debug("Cleanup done "+id);
									}};
							}}, c.throttle);
			trainerPool.shutdown();
			try {
				trainerPool.awaitTermination(7, TimeUnit.DAYS);
			} catch (InterruptedException e) {
				log.error("Interrupted?",e);
			}

			 /SrwTtwop */
        /* all diag tasks except SrwO: 
			Multithreading<String,PosNegRWExample> m = new Multithreading<String,PosNegRWExample>(log);
			m.executeJob(c.nthreads, new ParsedFile(groundedFile), 
					new Transformer<String,PosNegRWExample>() {
						@Override
						public Callable<PosNegRWExample> transformer(final String in, final int id) {
							return new Callable<PosNegRWExample>() {
								@Override
								public PosNegRWExample call() throws Exception {
									try {
									//log.debug("Job start "+id);
									//PosNegRWExample ret = parser.parse(in, b.copy());
									log.debug("Parsing start "+id);
									PosNegRWExample ret = new GroundedExampleParser().parse(in, b.copy());
									log.debug("Parsing done "+id);
									log.debug("Training start "+id);
									srw.trainOnExample(params,ret);
									log.debug("Training done "+id);
									//log.debug("Job done "+id);
									return ret;
									} catch (IllegalArgumentException e) {
										System.err.println("Problem with #"+id);
										e.printStackTrace();
									}
									return null;
								}};
						}}, new Cleanup<PosNegRWExample>() {
							@Override
							public Runnable cleanup(final Future<PosNegRWExample> in, final int id) {
								return new Runnable(){
									//ArrayList<PosNegRWExample> done = new ArrayList<PosNegRWExample>();
									@Override
									public void run() {
										try {
											//done.add(in.get());
											in.get();
										} catch (InterruptedException e) {
										    e.printStackTrace(); 
										} catch (ExecutionException e) {
										    e.printStackTrace();
										}
										log.debug("Cleanup start "+id);
										log.debug("Cleanup done "+id);
									}};
							}}, c.throttle);
			*/
        /* SrwO:
			   Multithreading<PosNegRWExample,Integer> m = new Multithreading<PosNegRWExample,Integer>(log);
			m.executeJob(c.nthreads, new PosNegRWExampleStreamer(new ParsedFile(groundedFile),new ArrayLearningGraphBuilder()), 
						 new Transformer<PosNegRWExample,Integer>() {
						@Override
						public Callable<Integer> transformer(final PosNegRWExample in, final int id) {
							return new Callable<Integer>() {
								@Override
								public Integer call() throws Exception {
									try {
									//log.debug("Job start "+id);
									//PosNegRWExample ret = parser.parse(in, b.copy());
									log.debug("Training start "+id);
									srw.trainOnExample(params,in);
									log.debug("Training done "+id);
									//log.debug("Job done "+id);
									} catch (IllegalArgumentException e) {
										System.err.println("Problem with #"+id);
										e.printStackTrace();
									}
									return in.length();
								}};
						}}, new Cleanup<Integer>() {
							@Override
							public Runnable cleanup(final Future<Integer> in, final int id) {
								return new Runnable(){
									//ArrayList<PosNegRWExample> done = new ArrayList<PosNegRWExample>();
									@Override
									public void run() {
										try {
											//done.add(in.get());
											in.get();
										} catch (InterruptedException e) {
										    e.printStackTrace(); 
										} catch (ExecutionException e) {
										    e.printStackTrace();
										}
										log.debug("Cleanup start "+id);
										log.debug("Cleanup done "+id);
									}};
							}}, c.throttle);
			*/
        srw.cleanupParams(params, params);
        log.info("Finished diagnostic in " + (System.currentTimeMillis() - start) + " ms");
    } catch (Throwable t) {
        t.printStackTrace();
        System.exit(-1);
    }
}