Examples with FstConfig org.apache.stanbol.entityhub.indexing.destination.solryard.fst.FstConfig used on opensource projects

Example 1 with FstConfig

use of org.apache.stanbol.entityhub.indexing.destination.solryard.fst.FstConfig in project stanbol by apache.

the class SolrYardIndexingDestination method parseFstConfig.

/**
     * @param lines
     */
private List<FstConfig> parseFstConfig(Collection<String> lines) {
    List<FstConfig> fstConfigs = new ArrayList<FstConfig>();
    for (String line : lines) {
        line = line.trim();
        if (!line.isEmpty() && line.charAt(0) != '#') {
            String[] fields = new String[] { null, null };
            int index = -1;
            for (String part : line.split("=|;")) {
                if (index >= 0) {
                    fields[index] = part;
                    index = -1;
                } else if ("index".equalsIgnoreCase(part)) {
                    index = 0;
                } else if ("store".equalsIgnoreCase(part)) {
                    index = 1;
                }
            }
            if (fields[0] == null) {
                throw new IllegalArgumentException("Invalid FST configuration " + "line: " + line + ". Param 'index={field}' is required " + "(syntax: 'index={field};store={field}', 'store is optional'')!");
            }
            fstConfigs.add(new FstConfig(fields[0], fields[1]));
        }
    }
    return fstConfigs;
}

Example 2 with FstConfig

use of org.apache.stanbol.entityhub.indexing.destination.solryard.fst.FstConfig in project stanbol by apache.

the class SolrYardIndexingDestination method finalise.

@SuppressWarnings("unchecked")
@Override
public void finalise() {
    //write the indexing configuration
    if (indexFieldConfiguration != null) {
        FieldMapper mapper = FieldMappingUtils.createDefaultFieldMapper(indexFieldConfiguration);
        try {
            CacheUtils.storeBaseMappingsConfiguration(solrYard, mapper);
        } catch (YardException e) {
            log.error("Unable to store FieldMapperConfiguration to the Store!", e);
        }
    }
    log.info(" ... optimize SolrCore");
    try {
        solrYard.optimize();
    } catch (YardException e) {
        log.error("Unable to optimize SolrIndex after indexing! IndexArchive will not be optimized ...", e);
    }
    //build the FST models
    if (fstConfigs != null) {
        //(1) FST config initialisation
        log.info(" ... init FST configuration(s)");
        IndexSchema schema = core.getLatestSchema();
        File fstDir = new File(new File(core.getDataDir()), "fst");
        if (!fstDir.isDirectory()) {
            try {
                FileUtils.forceMkdir(fstDir);
            } catch (IOException e) {
                throw new IllegalStateException("Unable to create Directory " + fstDir.getAbsolutePath() + "for storing the FST models " + "of SolrCore " + core.getName());
            }
        }
        RefCounted<SolrIndexSearcher> searcherRef = core.getSearcher();
        try {
            for (FstConfig fstConfig : fstConfigs) {
                //set the FST directory
                fstConfig.setFstDirectory(fstDir);
                log.info("> FST config {}", fstConfig);
                fstConfig.buildConfig(schema, searcherRef.get().getAtomicReader());
                for (CorpusCreationInfo corpus : fstConfig.getCorpusCreationInfos()) {
                    log.info("  - {}", corpus);
                }
            }
        } finally {
            searcherRef.decref();
        }
        List<Future<?>> fstCreationTasks = new ArrayList<Future<?>>();
        ExecutorService es = Executors.newFixedThreadPool(fstThreads);
        log.info(" ... build FST models ");
        for (FstConfig config : fstConfigs) {
            for (final CorpusCreationInfo corpus : config.getCorpusCreationInfos()) {
                fstCreationTasks.add(es.submit(new CorpusCreationTask(core, corpus)));
            }
        }
        //now wait for the completion of the tasks
        Iterator<Future<?>> taskIt = fstCreationTasks.iterator();
        while (taskIt.hasNext()) {
            Future<?> task = taskIt.next();
            try {
                //wait until ready
                task.get();
                taskIt.remove();
            } catch (ExecutionException e) {
                log.error("Exception while building FST models for SolrCore " + core.getName(), e);
            } catch (InterruptedException e) {
                log.error("Interupped while building FST models for SolrCore " + core.getName(), e);
                Thread.currentThread().interrupt();
            }
        }
        if (!fstCreationTasks.isEmpty()) {
            log.warn("Unable to build {} FST models for SolrCore {}", fstCreationTasks.size(), core.getName());
        } else {
            log.info("All FST modles for SolrCore {} build successfully!", core.getName());
        }
    }
    //no FST modles to build
    //all Solr specific stuff is now ready
    log.info(" ... close SolrCore");
    solrYard.close();
    //if a indexing config is present we need to create the distribution files
    if (indexingConfig != null) {
        //first check if the distribution folder needs to be created and is valid
        File distFolder = indexingConfig.getDistributionFolder();
        if (!distFolder.exists()) {
            if (!distFolder.mkdirs()) {
                throw new IllegalStateException("Unable to create distribution folder " + distFolder.getAbsolutePath());
            }
        } else if (!distFolder.isDirectory()) {
            throw new IllegalStateException("Distribution folder" + distFolder.getAbsolutePath() + "is not a Directory!");
        }
        //zip the index and copy it over to distribution
        log.info(" ... build Solr index archive");
        if (solrArchive != null) {
            try {
                writeSolrIndexArchive(indexingConfig);
            } catch (IOException e) {
                log.error("Error while creating Solr Archive " + solrArchive.getAbsolutePath() + "! The archive will not be created!", e);
                log.error("As a Workaround you can manually create the Solr Archive " + "by creating a ZIP archive with the contents of the Folder " + solrIndexLocation + "!");
            }
        }
        if (solrArchiveRef != null) {
            try {
                writeSolrIndexReference(indexingConfig);
            } catch (IOException e) {
                log.error("Error while creating Solr Archive Reference " + solrArchiveRef.getAbsolutePath() + "! The file will not be created!", e);
            }
        }
        //finally create the Osgi Configuration
        try {
            OsgiConfigurationUtil.writeSiteConfiguration(indexingConfig);
        } catch (IOException e) {
            log.error("Unable to write OSGI configuration file for the referenced site", e);
        }
        try {
            OsgiConfigurationUtil.writeCacheConfiguration(indexingConfig);
        } catch (IOException e) {
            log.error("Unable to write OSGI configuration file for the Cache", e);
        }
        //create the SolrYard configuration
        try {
            writeSolrYardConfiguration(indexingConfig);
        } catch (IOException e) {
            log.error("Unable to write OSGI configuration file for the SolrYard", e);
        }
        //create the bundle
        OsgiConfigurationUtil.createBundle(indexingConfig);
    }
}