Examples with TaxonMatcher org.opensextant.extractors.xtax.TaxonMatcher used on opensource projects

Example 1 with TaxonMatcher

use of org.opensextant.extractors.xtax.TaxonMatcher in project Xponents by OpenSextant.

the class PlaceGeocoder method configure.

/**
     * We do whatever is needed to init resources... that varies depending on
     * the use case.
     * 
     * Guidelines: this class is custodian of the app controller, Corpus feeder,
     * and any Document instances passed into/out of the feeder.
     * 
     * This geocoder requires a default /exclusions/person-name-filter.txt,
     * which can be empty, but most often it will be a list of person names
     * (which are non-place names)
     * 
     * Rules Configured in approximate order:
     * 
     * <pre>
     * CountryRule    -- tag all country names
     * NameCodeRule   -- parse any Name, CODE, or Name1, Name2 patterns for "Place, AdminPlace" evidence
     * PersonNameRule -- annotate, negate any patterns or matches that appear to be known celebrity persons or organizations.
     *                   Qualified places are not negated, e.g., "Euguene, Oregon" is a place;  "Euguene" with no other evidence is a person name.  
     * CoordRule      -- if requested, parse any coordinate patterns;  Reverse geocode Country + Province.
     * ProvinceAssociationRule  -- associate places with Province inferred by coordinates.
     * MajorPlaceRule -- identify major places by feature type, class or location population.
     * LocationChooserRule    -- final rule that assigns confidence and chooses best location(s)
     * 
     * Your Rule Here -- use addRule( GeocodeRule ) to add a rule on the stack.  It will be evaluated just before the final LocationChooserRule.
     * your rule should improve Place scores on PlaceCandidates and name the rules that fire.
     * </pre>
     *
     * @throws ConfigException
     *             on err
     */
@Override
public void configure() throws ConfigException {
    // ==============
    // Rule setup:  Create GeocodeRules, add them to this.rules if they can be evaluated generically
    // on a list of place tags.
    // Otherwise such rules are configured, set during the request, and evaluated adhoc as you need.
    // 
    /* assess country names and codes */
    countryRule = new CountryRule();
    countryRule.setCountryObserver(this);
    /* assess NAME, CODE patterns */
    nameWithAdminRule = new NameCodeRule();
    nameWithAdminRule.setBoundaryObserver(this);
    // Nonsense is filtered out, rather than scored and ranked low.
    nonsenseFilter = new NonsenseFilter();
    rules.add(nonsenseFilter);
    /**
         * Files for Place Name filter are editable, as you likely have
         * different ideas of who are "person names" to exclude when they
         * conflict with place names. If you are filtering out such things, then
         * it makes sense to filter them out earliest and not incorporate them
         * in geocoding.
         * 
         */
    personNameRule = new PersonNameFilter("/filters/person-name-filter.txt", "/filters/person-title-filter.txt", "/filters/person-suffix-filter.txt");
    rules.add(personNameRule);
    /*
         * assess coordinates related to ADM1, CC
         */
    coordRule = new CoordinateAssociationRule();
    coordRule.setCountryObserver(this);
    coordRule.setLocationObserver(this);
    if (xcoord == null && (isCoordExtractionEnabled())) {
        xcoord = new XCoord();
        xcoord.configure();
        /*
             * assess ADM1 related to found NAMES as a result of coordinates
             */
        adm1Rule = new ProvinceAssociationRule();
        adm1Rule.setCountryObserver(this);
        rules.add(coordRule);
        rules.add(adm1Rule);
    }
    //
    try {
        Map<String, Integer> popstats = GeonamesUtility.mapPopulationByLocation(GeonamesUtility.loadMajorCities("/geonames.org/cities15000.txt"));
        majorPlaceRule = new MajorPlaceRule(popstats);
    } catch (IOException err) {
        log.error("Xponents 2.8: cities population data is used for geocoding. Will continue without it.");
        majorPlaceRule = new MajorPlaceRule(null);
    }
    majorPlaceRule.setCountryObserver(this);
    majorPlaceRule.setBoundaryObserver(this);
    rules.add(majorPlaceRule);
    // 
    if (isPersonNameMatchingEnabled()) {
        try {
            personMatcher = new TaxonMatcher();
            personMatcher.configure();
            /*
                 * Default catalog must be built. Extraction ./XTax folder has
                 * script for populating a catalog.
                 */
            personMatcher.addCatalogFilter("JRC");
            personMatcher.addCatalogFilter("nationality");
        } catch (IOException err) {
            throw new ConfigException("XTax resource not available.");
        }
    }
    // Un-filter city names that can be resolved if other ADMIN places line up.
    // E.g., "Cleveland Caveliers" filters out Cleveland, but if Cleveland is mentioned alone
    // then Cleveland itself will be promoted to a location.  E.g., sports teams travel
    // so mention of "Cleveland Caveliers visiting Seattle" would not geolocate this to Ohio
    // unless the city or state was mentioned separately.  
    // 
    placeInOrgRule = new ContextualOrganizationRule();
    placeInOrgRule.setBoundaryObserver(this);
    rules.add(placeInOrgRule);
    // Simple patterns such as city of x or  abc county.
    // 
    rules.add(new NameRule());
    chooser = new LocationChooserRule();
    chooser.setCountryObserver(this);
    chooser.setBoundaryObserver(this);
    chooser.setLocationObserver(this);
    //rules.add(chooser);
    countryCatalog = this.getGazetteer().getCountries();
}

Example 2 with TaxonMatcher

use of org.opensextant.extractors.xtax.TaxonMatcher in project Xponents by OpenSextant.

the class TaxonomicTagger method main.

/**
     * Do a basic test of TaxTagger. Read a file, tag it.
     * 
     * @param args
     */
public static void main(String[] args) throws Exception {
    gnu.getopt.Getopt opts = new gnu.getopt.Getopt("TaxonomicTagger", args, "i:c:");
    int c = -1;
    String file = null;
    String catalog = null;
    while ((c = opts.getopt()) != -1) {
        switch(c) {
            case 'i':
                file = opts.getOptarg();
                break;
            case 'c':
                catalog = opts.getOptarg();
                break;
            default:
                System.out.println("Usage  -i filename [-c cat]");
                System.exit(-1);
        }
    }
    taxtag = new TaxonMatcher();
    taxtag.configure();
    try {
        if (new File(file).isDirectory()) {
            File[] files = new File(file).listFiles();
            for (File f : files) {
                System.out.println("\n\nFile = " + f + "\n=========MATCHES:=========");
                String doc = FileUtility.readFile(f);
                testDoc(doc);
            }
            return;
        }
        String doc = FileUtility.readFile(file);
        if (catalog != null) {
            // Invalid filter + valid filter.
            System.out.println("Testing a valid catalog - tagger will return only matches from subset catalog=" + catalog);
            taxtag.addCatalogFilter(catalog);
        }
        testDoc(doc);
    } catch (Exception err) {
        err.printStackTrace();
    }
    taxtag.cleanup();
}

Example 3 with TaxonMatcher

use of org.opensextant.extractors.xtax.TaxonMatcher in project Xponents by OpenSextant.

the class KeywordTaggerMapper method setup.

/**
     * Setup.  XTax or PlaceGecoder takes in SOLR path for xponents solr from JVM environment.
     */
@Override
public void setup(Context c) throws IOException {
    super.setup(c);
    try {
        xtax = new TaxonMatcher();
    } catch (ConfigException e) {
        // TODO Auto-generated catch block
        throw new IOException("setup.XTax", e);
    }
    log.info("DONE");
}

Example 4 with TaxonMatcher

use of org.opensextant.extractors.xtax.TaxonMatcher in project Xponents by OpenSextant.

the class TestXTax method main.

/**
     * 
     *
     * @param args
     *            the arguments
     * @throws Exception
     *             the exception
     */
public static void main(String[] args) {
    TaxonMatcher tax = null;
    try {
        tax = new TaxonMatcher();
        // Find JRC entities that have this random id pattern ID='1*1' and are in Russian form.
        // 
        List<Taxon> results = tax.search("tag:jrc_id+1*1 AND tag:lang_id+ru");
        for (Taxon tx : results) {
            System.out.println("Found: " + getJRCTag(tx.tagset) + " = " + tx);
        }
        if (args.length > 0) {
            File f = new File(args[0]);
            String content = FileUtility.readFile(f, "UTF-8");
            List<TextMatch> findings = tax.extract(content);
            for (TextMatch tm : findings) {
                String type = "" + ((TaxonMatch) tm).getTaxons();
                System.out.println("Found: " + tm + "\n\t\t" + type);
            }
        }
    } catch (Exception err) {
        err.printStackTrace();
    } finally {
        tax.shutdown();
        System.exit(0);
    }
}