Examples with FSA com.yahoo.fsa.FSA used on opensource projects

Example 1 with FSA

use of com.yahoo.fsa.FSA in project vespa by vespa-engine.

the class RewriterUtils method getRewriteFromFSA.

/**
 * Retrieve rewrite from FSA given the original query
 *
 * @param query Query object from searcher
 * @param dictName FSA dictionary name
 * @param rewriterDicts list of rewriter dictionaries
 *                      It has the following format:
 *                      HashMap<dictionary name, FSA>
 * @param key The original query used to retrieve rewrite
 *            from the dictionary
 * @return String The retrieved rewrites, null if query
 *         doesn't exist
 */
public static String getRewriteFromFSA(Query query, HashMap<String, Object> rewriterDicts, String dictName, String key) throws RuntimeException {
    if (rewriterDicts == null) {
        error(utilsLogger, query, "HashMap containing rewriter dicts is null");
        throw new RuntimeException("HashMap containing rewriter dicts is null");
    }
    FSA fsa = (FSA) rewriterDicts.get(dictName);
    if (fsa == null) {
        error(utilsLogger, query, "Error retrieving FSA dictionary: " + dictName);
        throw new RuntimeException("Error retrieving FSA dictionary: " + dictName);
    }
    String result = null;
    result = fsa.lookup(key);
    log(utilsLogger, query, "Retrieved rewrite: " + result);
    return result;
}

Example 2 with FSA

use of com.yahoo.fsa.FSA in project vespa by vespa-engine.

the class RewriterUtils method loadFSA.

/**
 * Load FSA from file
 *
 * @param file FSA dictionary file object
 * @param query Query object from the searcher, could be null if not available
 * @return FSA The FSA object for the input file path
 */
public static FSA loadFSA(File file, Query query) throws IOException {
    log(utilsLogger, query, "Loading FSA file");
    String filePath = null;
    try {
        filePath = file.getAbsolutePath();
    } catch (SecurityException e1) {
        error(utilsLogger, query, "No read access for the FSA file");
        throw new IOException("No read access for the FSA file");
    }
    FSA fsa = loadFSA(filePath, query);
    return fsa;
}

Example 3 with FSA

use of com.yahoo.fsa.FSA in project vespa by vespa-engine.

the class QueryRewriteSearcher method loadFSADicts.

/**
 * Load the dicts specified in vespa-services.xml
 *
 * @param fileAcquirer Required param for retrieving file type config
 *                     (see vespa's search container doc for more detail)
 * @param config Config from vespa-services.xml (see vespa's search
 *               container doc for more detail)
 * @param fileList pairs of file name and file handler for unit tests
 * @return boolean true if loaded successfully, false otherwise
 */
private boolean loadFSADicts(FileAcquirer fileAcquirer, RewritesConfig config, HashMap<String, File> fileList) throws RuntimeException {
    // Check if getRewriterName method is properly implemented
    String rewriterName = getRewriterName();
    if (rewriterName == null) {
        RewriterUtils.error(logger, "Rewriter required method is not properly implemented: ");
        return false;
    }
    RewriterUtils.log(logger, "Configuring rewriter: " + rewriterName);
    // Check if there's no config need to be loaded
    if (config == null || (fileAcquirer == null && fileList == null)) {
        RewriterUtils.log(logger, "No FSA dictionary file need to be loaded");
        return true;
    }
    // Check if config contains the FSADict param
    if (config.fsaDict() == null) {
        RewriterUtils.error(logger, "FSADict is not properly set in config");
        return false;
    }
    RewriterUtils.log(logger, "Loading rewriter dictionaries");
    // Retrieve FSA names and paths
    ListIterator<FsaDict> fsaList = config.fsaDict().listIterator();
    // Load default dictionaries if no user dictionaries is configured
    if (!fsaList.hasNext()) {
        RewriterUtils.log(logger, "Loading default dictionaries");
        HashMap<String, String> defaultFSAs = getDefaultFSAs();
        if (defaultFSAs == null) {
            RewriterUtils.log(logger, "No default FSA dictionary is configured");
            return true;
        }
        Iterator<Map.Entry<String, String>> defaultFSAList = defaultFSAs.entrySet().iterator();
        while (defaultFSAList.hasNext()) {
            try {
                Map.Entry<String, String> currFSA = defaultFSAList.next();
                String fsaName = currFSA.getKey();
                String fsaPath = currFSA.getValue();
                RewriterUtils.log(logger, "FSA file location for " + fsaName + ": " + fsaPath);
                // Load FSA
                FSA fsa = RewriterUtils.loadFSA(RewriterConstants.DEFAULT_DICT_DIR + fsaPath, null);
                // Store FSA into dictionary map
                rewriterDicts.put(fsaName, fsa);
            } catch (IOException e) {
                RewriterUtils.error(logger, "Error loading FSA dictionary: " + e.getMessage());
                return false;
            }
        }
    } else {
        // Load user configured dictionaries
        while (fsaList.hasNext()) {
            try {
                FsaDict currFSA = fsaList.next();
                // fsaName and fsaPath are not null
                // or else vespa config server would not have been
                // able to start up
                String fsaName = currFSA.name();
                FileReference fsaPath = currFSA.path();
                RewriterUtils.log(logger, "FSA file location for " + fsaName + ": " + fsaPath);
                // Retrieve FSA File handler
                File fsaFile = null;
                if (fileAcquirer != null) {
                    fsaFile = fileAcquirer.waitFor(fsaPath, 5, TimeUnit.MINUTES);
                } else if (fileList != null) {
                    fsaFile = fileList.get(fsaName);
                }
                if (fsaFile == null) {
                    RewriterUtils.error(logger, "Error loading FSA dictionary file handler");
                    return false;
                }
                // Load FSA
                FSA fsa = RewriterUtils.loadFSA(fsaFile, null);
                // Store FSA into dictionary map
                rewriterDicts.put(fsaName, fsa);
            } catch (InterruptedException e1) {
                RewriterUtils.error(logger, "Error loading FSA dictionary file handler: " + e1.getMessage());
                return false;
            } catch (IOException e2) {
                RewriterUtils.error(logger, "Error loading FSA dictionary: " + e2.getMessage());
                return false;
            }
        }
    }
    RewriterUtils.log(logger, "Successfully loaded rewriter dictionaries");
    return true;
}

Example 4 with FSA

use of com.yahoo.fsa.FSA in project vespa by vespa-engine.

the class GenericExpansionRewriter method configure.

/**
 * Instance creation time config loading besides FSA.
 * Create PhraseMatcher from FSA dict
 */
public boolean configure(FileAcquirer fileAcquirer, RewritesConfig config, HashMap<String, File> fileList) {
    logger = Logger.getLogger(GenericExpansionRewriter.class.getName());
    FSA fsa = (FSA) rewriterDicts.get(GENERIC_EXPAND_DICT);
    if (fsa == null) {
        RewriterUtils.error(logger, "Error retrieving FSA dictionary: " + GENERIC_EXPAND_DICT);
        return false;
    }
    // Create Phrase Matcher
    RewriterUtils.log(logger, "Creating PhraseMatcher");
    try {
        phraseMatcher = new PhraseMatcher(fsa, false);
    } catch (IllegalArgumentException e) {
        RewriterUtils.error(logger, "Error creating phrase matcher");
        return false;
    }
    // Match single word as well
    phraseMatcher.setMatchSingleItems(true);
    // Return all matches instead of only the longest match
    phraseMatcher.setMatchAll(true);
    return true;
}

Example 5 with FSA

use of com.yahoo.fsa.FSA in project vespa by vespa-engine.

the class RewriterUtils method loadFSA.

/**
 * Load FSA from file
 *
 * @param filename FSA dictionary file path
 * @param query Query object from the searcher, could be null if not available
 * @return FSA The FSA object for the input file path
 */
public static FSA loadFSA(String filename, Query query) throws IOException {
    log(utilsLogger, query, "Loading FSA file from: " + filename);
    if (!new File(filename).exists()) {
        error(utilsLogger, query, "File does not exist : " + filename);
        throw new IOException("File does not exist : " + filename);
    }
    FSA fsa;
    try {
        fsa = new FSA(filename);
    } catch (RuntimeException e) {
        error(utilsLogger, query, "Invalid FSA file");
        throw new IOException("Invalid FSA file");
    }
    if (!fsa.isOk()) {
        error(utilsLogger, query, "Unable to load FSA file from : " + filename);
        throw new IOException("Not able to load FSA file from : " + filename);
    }
    log(utilsLogger, query, "Loaded FSA successfully from file : " + filename);
    return fsa;
}