Examples with CorpusCreationTask org.apache.stanbol.entityhub.indexing.destination.solryard.fst.CorpusCreationTask used on opensource projects

Example 1 with CorpusCreationTask

use of org.apache.stanbol.entityhub.indexing.destination.solryard.fst.CorpusCreationTask in project stanbol by apache.

the class SolrYardIndexingDestination method finalise.

@SuppressWarnings("unchecked")
@Override
public void finalise() {
    // write the indexing configuration
    if (indexFieldConfiguration != null) {
        FieldMapper mapper = FieldMappingUtils.createDefaultFieldMapper(indexFieldConfiguration);
        try {
            CacheUtils.storeBaseMappingsConfiguration(solrYard, mapper);
        } catch (YardException e) {
            log.error("Unable to store FieldMapperConfiguration to the Store!", e);
        }
    }
    log.info(" ... optimize SolrCore");
    try {
        solrYard.optimize();
    } catch (YardException e) {
        log.error("Unable to optimize SolrIndex after indexing! IndexArchive will not be optimized ...", e);
    }
    // build the FST models
    if (fstConfigs != null) {
        // (1) FST config initialisation
        log.info(" ... init FST configuration(s)");
        IndexSchema schema = core.getLatestSchema();
        File fstDir = new File(new File(core.getDataDir()), "fst");
        if (!fstDir.isDirectory()) {
            try {
                FileUtils.forceMkdir(fstDir);
            } catch (IOException e) {
                throw new IllegalStateException("Unable to create Directory " + fstDir.getAbsolutePath() + "for storing the FST models " + "of SolrCore " + core.getName());
            }
        }
        RefCounted<SolrIndexSearcher> searcherRef = core.getSearcher();
        try {
            for (FstConfig fstConfig : fstConfigs) {
                // set the FST directory
                fstConfig.setFstDirectory(fstDir);
                log.info("> FST config {}", fstConfig);
                fstConfig.buildConfig(schema, searcherRef.get().getAtomicReader());
                for (CorpusCreationInfo corpus : fstConfig.getCorpusCreationInfos()) {
                    log.info("  - {}", corpus);
                }
            }
        } finally {
            searcherRef.decref();
        }
        List<Future<?>> fstCreationTasks = new ArrayList<Future<?>>();
        ExecutorService es = Executors.newFixedThreadPool(fstThreads);
        log.info(" ... build FST models ");
        for (FstConfig config : fstConfigs) {
            for (final CorpusCreationInfo corpus : config.getCorpusCreationInfos()) {
                fstCreationTasks.add(es.submit(new CorpusCreationTask(core, corpus)));
            }
        }
        // now wait for the completion of the tasks
        Iterator<Future<?>> taskIt = fstCreationTasks.iterator();
        while (taskIt.hasNext()) {
            Future<?> task = taskIt.next();
            try {
                // wait until ready
                task.get();
                taskIt.remove();
            } catch (ExecutionException e) {
                log.error("Exception while building FST models for SolrCore " + core.getName(), e);
            } catch (InterruptedException e) {
                log.error("Interupped while building FST models for SolrCore " + core.getName(), e);
                Thread.currentThread().interrupt();
            }
        }
        if (!fstCreationTasks.isEmpty()) {
            log.warn("Unable to build {} FST models for SolrCore {}", fstCreationTasks.size(), core.getName());
        } else {
            log.info("All FST modles for SolrCore {} build successfully!", core.getName());
        }
    }
    // no FST modles to build
    // all Solr specific stuff is now ready
    log.info(" ... close SolrCore");
    solrYard.close();
    // if a indexing config is present we need to create the distribution files
    if (indexingConfig != null) {
        // first check if the distribution folder needs to be created and is valid
        File distFolder = indexingConfig.getDistributionFolder();
        if (!distFolder.exists()) {
            if (!distFolder.mkdirs()) {
                throw new IllegalStateException("Unable to create distribution folder " + distFolder.getAbsolutePath());
            }
        } else if (!distFolder.isDirectory()) {
            throw new IllegalStateException("Distribution folder" + distFolder.getAbsolutePath() + "is not a Directory!");
        }
        // zip the index and copy it over to distribution
        log.info(" ... build Solr index archive");
        if (solrArchive != null) {
            try {
                writeSolrIndexArchive(indexingConfig);
            } catch (IOException e) {
                log.error("Error while creating Solr Archive " + solrArchive.getAbsolutePath() + "! The archive will not be created!", e);
                log.error("As a Workaround you can manually create the Solr Archive " + "by creating a ZIP archive with the contents of the Folder " + solrIndexLocation + "!");
            }
        }
        if (solrArchiveRef != null) {
            try {
                writeSolrIndexReference(indexingConfig);
            } catch (IOException e) {
                log.error("Error while creating Solr Archive Reference " + solrArchiveRef.getAbsolutePath() + "! The file will not be created!", e);
            }
        }
        // finally create the Osgi Configuration
        try {
            OsgiConfigurationUtil.writeSiteConfiguration(indexingConfig);
        } catch (IOException e) {
            log.error("Unable to write OSGI configuration file for the referenced site", e);
        }
        try {
            OsgiConfigurationUtil.writeCacheConfiguration(indexingConfig);
        } catch (IOException e) {
            log.error("Unable to write OSGI configuration file for the Cache", e);
        }
        // create the SolrYard configuration
        try {
            writeSolrYardConfiguration(indexingConfig);
        } catch (IOException e) {
            log.error("Unable to write OSGI configuration file for the SolrYard", e);
        }
        // create the bundle
        OsgiConfigurationUtil.createBundle(indexingConfig);
    }
}