use of gate.DataStore in project gate-core by GateNLP.
the class MainFrame method openDocServiceDataStore.
// openSerialDataStore()
/**
* Method is used in ....OpenDSAction
* @return the opened datastore or null if an error occurs
*/
protected DataStore openDocServiceDataStore() {
DataStore ds = null;
try {
String DSLocation = JOptionPane.showInputDialog(MainFrame.this, "Enter document service URL", "http://localhost:8080/docservice/services/docservice");
// System.out.println("DEBUG: MainFrame.openDocServiceDataStore()
// DSLocation='" + DSLocation + "'");
ds = Factory.openDataStore("gleam.docservice.gate.DocServiceDataStore", DSLocation);
} catch (Exception error) {
String message = "Error when opening the Datastore.";
log.error(message, error);
}
return ds;
}
use of gate.DataStore in project gate-core by GateNLP.
the class MainFrame method createSearchableDataStore.
/**
* Method is used in NewDSAction
* @return the new datastore or null if an error occurs
*/
protected DataStore createSearchableDataStore() {
try {
JPanel mainPanel = new JPanel(new GridBagLayout());
final JTextField dsLocation = new JTextField("", 20);
dsLocation.setEditable(false);
final JTextField indexLocation = new JTextField("", 20);
indexLocation.setToolTipText("directory to store the the lucene index");
JTextField btat = new JTextField("Token", 20);
btat.setToolTipText("Examples: Token, AnnotationSetName.Token, " + Constants.DEFAULT_ANNOTATION_SET_NAME + ".Token");
JCheckBox createTokensAutomatically = new JCheckBox("Create Tokens Automatically");
createTokensAutomatically.setSelected(true);
JTextField iuat = new JTextField("", 20);
iuat.setToolTipText("Examples: Sentence, AnnotationSetName.Sentence, " + Constants.DEFAULT_ANNOTATION_SET_NAME + ".Sentence");
final List<String> inputASList = new ArrayList<String>();
inputASList.add("Key");
inputASList.add(Constants.DEFAULT_ANNOTATION_SET_NAME);
final JTextField inputAS = new JTextField("", 20);
inputAS.setText("Key;" + Constants.DEFAULT_ANNOTATION_SET_NAME);
inputAS.setEditable(false);
JButton editInputAS = new JButton(getIcon("edit-list"));
editInputAS.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent ae) {
ListEditorDialog listEditor = new ListEditorDialog(instance, inputASList, "java.lang.String");
@SuppressWarnings("unchecked") List<String> result = listEditor.showDialog();
if (result != null) {
inputASList.clear();
inputASList.addAll(result);
if (inputASList.size() > 0) {
String text = inputASList.get(0) == null ? Constants.DEFAULT_ANNOTATION_SET_NAME : inputASList.get(0);
for (int j = 1; j < inputASList.size(); j++) {
text += ";" + (inputASList.get(j) == null ? Constants.DEFAULT_ANNOTATION_SET_NAME : inputASList.get(j));
}
inputAS.setText(text);
} else {
inputAS.setText("");
}
}
}
});
JComboBox<String> asie = new JComboBox<String>(new String[] { "include", "exclude" });
inputAS.setToolTipText("Leave blank for indexing all annotation sets. \"" + Constants.DEFAULT_ANNOTATION_SET_NAME + "\" indicates the default annotation set");
final List<String> fteList = new ArrayList<String>();
fteList.add("SpaceToken");
fteList.add("Split");
final JTextField fte = new JTextField("", 20);
fte.setText("SpaceToken;Split");
fte.setEditable(false);
JButton editFTE = new JButton(getIcon("edit-list"));
editFTE.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent ae) {
ListEditorDialog listEditor = new ListEditorDialog(instance, fteList, "java.lang.String");
@SuppressWarnings("unchecked") List<String> result = listEditor.showDialog();
if (result != null) {
fteList.clear();
fteList.addAll(result);
if (fteList.size() > 0) {
String text = fteList.get(0) == null ? Constants.DEFAULT_ANNOTATION_SET_NAME : fteList.get(0);
for (int j = 1; j < fteList.size(); j++) {
text += ";" + (fteList.get(j) == null ? Constants.DEFAULT_ANNOTATION_SET_NAME : fteList.get(j));
}
fte.setText(text);
} else {
fte.setText("");
}
}
}
});
JComboBox<String> ftie = new JComboBox<String>(new String[] { "include", "exclude" });
ftie.setSelectedIndex(1);
fte.setToolTipText("Leave blank for inclusion of all features");
JButton indexBrowse = new JButton(getIcon("open-file"));
indexBrowse.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent ae) {
// first we need to ask for a new empty directory
fileChooser.setDialogTitle("Please create a new empty directory for datastore");
fileChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
fileChooser.setResource("gate.DataStore.index");
if (fileChooser.showOpenDialog(MainFrame.this) == JFileChooser.APPROVE_OPTION) {
try {
indexLocation.setText(fileChooser.getSelectedFile().toURI().toURL().toExternalForm());
} catch (Exception e) {
indexLocation.setText("");
}
}
}
});
GridBagConstraints constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 0;
constraints.gridwidth = 3;
constraints.anchor = GridBagConstraints.WEST;
constraints.fill = GridBagConstraints.NONE;
constraints.insets = new Insets(0, 0, 0, 5);
mainPanel.add(new JLabel("Datastore URL:"), constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 0;
constraints.gridwidth = 6;
constraints.fill = GridBagConstraints.HORIZONTAL;
constraints.insets = new Insets(0, 0, 0, 10);
mainPanel.add(dsLocation, constraints);
// second row
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 1;
constraints.gridwidth = 3;
constraints.anchor = GridBagConstraints.WEST;
constraints.fill = GridBagConstraints.NONE;
constraints.insets = new Insets(0, 0, 0, 5);
mainPanel.add(new JLabel("Index Location:"), constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 1;
constraints.gridwidth = 5;
constraints.fill = GridBagConstraints.HORIZONTAL;
constraints.insets = new Insets(0, 0, 0, 10);
mainPanel.add(indexLocation, constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 1;
constraints.gridwidth = 1;
constraints.anchor = GridBagConstraints.NORTHWEST;
mainPanel.add(indexBrowse, constraints);
indexBrowse.setBorderPainted(false);
indexBrowse.setContentAreaFilled(false);
// third row row
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 2;
constraints.gridwidth = 2;
constraints.anchor = GridBagConstraints.WEST;
constraints.fill = GridBagConstraints.NONE;
constraints.insets = new Insets(0, 0, 0, 5);
mainPanel.add(new JLabel("Annotation Sets:"), constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 2;
constraints.gridwidth = 1;
constraints.fill = GridBagConstraints.HORIZONTAL;
constraints.insets = new Insets(0, 0, 0, 10);
mainPanel.add(asie, constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 2;
constraints.gridwidth = 5;
constraints.fill = GridBagConstraints.HORIZONTAL;
constraints.insets = new Insets(0, 0, 0, 10);
mainPanel.add(inputAS, constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 2;
constraints.gridwidth = 1;
constraints.anchor = GridBagConstraints.NORTHWEST;
mainPanel.add(editInputAS, constraints);
editInputAS.setBorderPainted(false);
editInputAS.setContentAreaFilled(false);
// fourth row row
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 3;
constraints.gridwidth = 3;
constraints.anchor = GridBagConstraints.WEST;
constraints.fill = GridBagConstraints.NONE;
constraints.insets = new Insets(0, 0, 0, 5);
mainPanel.add(new JLabel("Base Token Type:"), constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 3;
constraints.gridwidth = 5;
constraints.anchor = GridBagConstraints.NORTHWEST;
mainPanel.add(btat, constraints);
// fifth row
constraints = new GridBagConstraints();
constraints.gridx = 4;
constraints.gridy = 4;
constraints.gridwidth = 5;
constraints.anchor = GridBagConstraints.WEST;
constraints.fill = GridBagConstraints.NONE;
constraints.insets = new Insets(0, 0, 0, 5);
mainPanel.add(createTokensAutomatically, constraints);
// sixth row
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 5;
constraints.gridwidth = 3;
constraints.anchor = GridBagConstraints.WEST;
constraints.fill = GridBagConstraints.NONE;
constraints.insets = new Insets(0, 0, 0, 5);
mainPanel.add(new JLabel("Index Unit Type:"), constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 5;
constraints.gridwidth = 5;
constraints.anchor = GridBagConstraints.NORTHWEST;
mainPanel.add(iuat, constraints);
// seventh row
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 6;
constraints.gridwidth = 2;
constraints.anchor = GridBagConstraints.WEST;
constraints.fill = GridBagConstraints.NONE;
constraints.insets = new Insets(0, 0, 0, 5);
mainPanel.add(new JLabel("Features:"), constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 6;
constraints.gridwidth = 1;
constraints.fill = GridBagConstraints.HORIZONTAL;
constraints.insets = new Insets(0, 0, 0, 10);
mainPanel.add(ftie, constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 6;
constraints.gridwidth = 5;
constraints.fill = GridBagConstraints.HORIZONTAL;
constraints.insets = new Insets(0, 0, 0, 10);
mainPanel.add(fte, constraints);
constraints = new GridBagConstraints();
constraints.gridx = GridBagConstraints.RELATIVE;
constraints.gridy = 6;
constraints.gridwidth = 1;
constraints.anchor = GridBagConstraints.NORTHWEST;
mainPanel.add(editFTE, constraints);
editFTE.setBorderPainted(false);
editFTE.setContentAreaFilled(false);
// get the URL (a folder in this case)
fileChooser.setDialogTitle("Please create a new empty directory");
fileChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
fileChooser.setFileFilter(fileChooser.getAcceptAllFileFilter());
fileChooser.setResource("gate.persist.LuceneDataStoreImpl");
int response = fileChooser.showOpenDialog(MainFrame.this);
if (response == JFileChooser.APPROVE_OPTION) {
try {
File dsFolder = fileChooser.getSelectedFile();
dsLocation.setText(dsFolder.toURI().toURL().toExternalForm());
File indexFolder = new File(dsFolder.getParentFile(), dsFolder.getName() + "-index");
indexLocation.setText(indexFolder.toURI().toURL().toExternalForm());
} catch (MalformedURLException mue) {
JOptionPane.showMessageDialog(MainFrame.this, "Invalid location\n " + mue.toString(), "GATE", JOptionPane.ERROR_MESSAGE);
}
} else {
return null;
}
boolean validEntry = false;
while (!validEntry) {
int returnValue = JOptionPane.showOptionDialog(MainFrame.this, mainPanel, "SearchableDataStore", JOptionPane.PLAIN_MESSAGE, JOptionPane.OK_CANCEL_OPTION, getIcon("empty"), new String[] { "OK", "Cancel" }, "OK");
if (returnValue == JOptionPane.OK_OPTION) {
if (dsLocation.getText().equals(indexLocation.getText())) {
JOptionPane.showMessageDialog(MainFrame.this, "Datastore and index cannot be stored in the same directory", "Error", JOptionPane.ERROR_MESSAGE);
} else {
// check if index folder can be created
try {
File indexDir = new File(new URL(indexLocation.getText()).getFile());
if (indexDir.exists() && indexDir.isFile()) {
JOptionPane.showMessageDialog(MainFrame.this, indexDir.getAbsolutePath() + " is a file on your disk. Index directory must be an" + " empty folder.", "Error", JOptionPane.ERROR_MESSAGE);
continue;
} else if (indexDir.isDirectory() && indexDir.list().length > 0) {
JOptionPane.showMessageDialog(instance, "Index directory " + indexDir.getAbsolutePath() + " must be an empty folder. ", "Error", JOptionPane.ERROR_MESSAGE);
continue;
} else {
if (!indexDir.exists()) {
if (!indexDir.mkdirs()) {
JOptionPane.showMessageDialog(MainFrame.this, "Cannot create index directory " + indexDir.getAbsolutePath() + "an empty folder. ", "Error", JOptionPane.ERROR_MESSAGE);
continue;
}
}
}
} catch (MalformedURLException mue) {
JOptionPane.showMessageDialog(MainFrame.this, "Invalid index location " + indexLocation.getText(), "Error", JOptionPane.ERROR_MESSAGE);
continue;
} catch (SecurityException se) {
JOptionPane.showMessageDialog(MainFrame.this, "Could not create a directory " + indexLocation.getText() + " because " + se.getMessage(), "Error", JOptionPane.ERROR_MESSAGE);
continue;
}
// if here.. an empty index directory exists
// break the loop by setting validEntry to true
validEntry = true;
DataStore ds = Factory.createDataStore("gate.persist.LuceneDataStoreImpl", dsLocation.getText());
// we need to set Indexer
Class<?>[] consParam = new Class<?>[1];
consParam[0] = URL.class;
Constructor<?> constructor = Class.forName("gate.creole.annic.lucene.LuceneIndexer", true, Gate.getClassLoader()).getConstructor(consParam);
Object indexer = constructor.newInstance(new URL(indexLocation.getText()));
Map<String, Object> parameters = new HashMap<String, Object>();
parameters.put(Constants.INDEX_LOCATION_URL, new URL(indexLocation.getText()));
parameters.put(Constants.BASE_TOKEN_ANNOTATION_TYPE, btat.getText());
parameters.put(Constants.INDEX_UNIT_ANNOTATION_TYPE, iuat.getText());
parameters.put(Constants.CREATE_TOKENS_AUTOMATICALLY, createTokensAutomatically.isSelected());
if (inputAS.getText().trim().length() > 0) {
ArrayList<String> inputASList1 = new ArrayList<String>();
String[] inputASArray = inputAS.getText().trim().split(";");
if (inputASArray != null && inputASArray.length > 0) {
inputASList1.addAll(Arrays.asList(inputASArray));
}
if (asie.getSelectedIndex() == 0) {
// user has provided values for inclusion
parameters.put(Constants.ANNOTATION_SETS_NAMES_TO_INCLUDE, inputASList1);
parameters.put(Constants.ANNOTATION_SETS_NAMES_TO_EXCLUDE, new ArrayList<String>());
} else {
// user has provided values for exclusion
parameters.put(Constants.ANNOTATION_SETS_NAMES_TO_EXCLUDE, inputASList1);
parameters.put(Constants.ANNOTATION_SETS_NAMES_TO_INCLUDE, new ArrayList<String>());
}
} else {
parameters.put(Constants.ANNOTATION_SETS_NAMES_TO_EXCLUDE, new ArrayList<String>());
parameters.put(Constants.ANNOTATION_SETS_NAMES_TO_INCLUDE, new ArrayList<String>());
}
if (fte.getText().trim().length() > 0) {
ArrayList<String> fteList1 = new ArrayList<String>();
String[] inputASArray = fte.getText().trim().split(";");
if (inputASArray != null && inputASArray.length > 0) {
fteList1.addAll(Arrays.asList(inputASArray));
}
if (ftie.getSelectedIndex() == 0) {
// user has provided values for inclusion
parameters.put(Constants.FEATURES_TO_INCLUDE, fteList1);
parameters.put(Constants.FEATURES_TO_EXCLUDE, new ArrayList<String>());
} else {
// user has provided values for exclusion
parameters.put(Constants.FEATURES_TO_EXCLUDE, fteList1);
parameters.put(Constants.FEATURES_TO_INCLUDE, new ArrayList<String>());
}
} else {
parameters.put(Constants.FEATURES_TO_EXCLUDE, new ArrayList<String>());
parameters.put(Constants.FEATURES_TO_INCLUDE, new ArrayList<String>());
}
Class<?>[] params = new Class<?>[2];
params[0] = Class.forName("gate.creole.annic.Indexer", true, Gate.getClassLoader());
params[1] = Map.class;
Method indexerMethod = ds.getClass().getMethod("setIndexer", params);
indexerMethod.invoke(ds, indexer, parameters);
// Class[] searchConsParams = new Class[0];
Constructor<?> searcherConst = Class.forName("gate.creole.annic.lucene.LuceneSearcher", true, Gate.getClassLoader()).getConstructor();
Object searcher = searcherConst.newInstance();
Class<?>[] searchParams = new Class<?>[1];
searchParams[0] = Class.forName("gate.creole.annic.Searcher", true, Gate.getClassLoader());
Method searcherMethod = ds.getClass().getMethod("setSearcher", searchParams);
searcherMethod.invoke(ds, searcher);
return ds;
}
} else {
validEntry = true;
}
}
return null;
} catch (Exception e) {
throw new GateRuntimeException(e);
}
}
use of gate.DataStore in project gate-core by GateNLP.
the class CorpusBenchmarkTool method evaluateMarkedStored.
// evaluateCorpus
protected void evaluateMarkedStored(File markedDir, File storedDir, File errDir) {
Document persDoc = null;
Document cleanDoc = null;
Document markedDoc = null;
// open the datastore and process each document
try {
// open the data store
DataStore sds = Factory.openDataStore("gate.persist.SerialDataStore", storedDir.toURI().toURL().toExternalForm());
List<String> lrIDs = sds.getLrIds("gate.corpora.DocumentImpl");
for (int i = 0; i < lrIDs.size(); i++) {
String docID = lrIDs.get(i);
// read the stored document
FeatureMap features = Factory.newFeatureMap();
features.put(DataStore.DATASTORE_FEATURE_NAME, sds);
features.put(DataStore.LR_ID_FEATURE_NAME, docID);
FeatureMap hparams = Factory.newFeatureMap();
// Gate.setHiddenAttribute(hparams, true);
persDoc = (Document) Factory.createResource("gate.corpora.DocumentImpl", features, hparams);
if (isMoreInfoMode) {
StringBuffer errName = new StringBuffer(persDoc.getName());
errName.replace(persDoc.getName().lastIndexOf("."), persDoc.getName().length(), ".err");
Out.prln("<H2>" + "<a href=\"err/" + errName.toString() + "\">" + persDoc.getName() + "</a>" + "</H2>");
} else
Out.prln("<H2>" + persDoc.getName() + "</H2>");
if (!this.isMarkedDS) {
// try finding the marked document as file
StringBuffer docName = new StringBuffer(persDoc.getName());
docName.replace(persDoc.getName().lastIndexOf("."), docName.length(), ".xml");
File markedDocFile = new File(markedDir, docName.toString());
if (!markedDocFile.exists()) {
Out.prln("Warning: Cannot find human-annotated document " + markedDocFile + " in " + markedDir);
} else {
FeatureMap params = Factory.newFeatureMap();
params.put(Document.DOCUMENT_URL_PARAMETER_NAME, markedDocFile.toURI().toURL());
params.put(Document.DOCUMENT_ENCODING_PARAMETER_NAME, documentEncoding);
// create the document
markedDoc = (Document) Factory.createResource("gate.corpora.DocumentImpl", params, hparams);
markedDoc.setName(persDoc.getName());
}
// find marked as file
} else {
try {
// open marked from a DS
// open the data store
DataStore sds1 = Factory.openDataStore("gate.persist.SerialDataStore", markedDir.toURI().toURL().toExternalForm());
List<String> lrIDs1 = sds1.getLrIds("gate.corpora.DocumentImpl");
boolean found = false;
int k = 0;
// search for the marked doc with the same name
while (k < lrIDs1.size() && !found) {
String docID1 = lrIDs1.get(k);
// read the stored document
FeatureMap features1 = Factory.newFeatureMap();
features1.put(DataStore.DATASTORE_FEATURE_NAME, sds1);
features1.put(DataStore.LR_ID_FEATURE_NAME, docID1);
Document tempDoc = (Document) Factory.createResource("gate.corpora.DocumentImpl", features1, hparams);
// check whether this is our doc
if (((String) tempDoc.getFeatures().get("gate.SourceURL")).endsWith(persDoc.getName())) {
found = true;
markedDoc = tempDoc;
} else
k++;
}
} catch (java.net.MalformedURLException ex) {
Out.prln("Error finding marked directory " + markedDir.getAbsolutePath());
} catch (gate.persist.PersistenceException ex1) {
Out.prln("Error opening marked as a datastore (-marked_ds specified)");
} catch (gate.creole.ResourceInstantiationException ex2) {
Out.prln("Error opening marked as a datastore (-marked_ds specified)");
}
}
evaluateDocuments(persDoc, cleanDoc, markedDoc, errDir);
if (persDoc != null) {
final gate.Document pd = persDoc;
javax.swing.SwingUtilities.invokeLater(new Runnable() {
@Override
public void run() {
Factory.deleteResource(pd);
}
});
}
if (markedDoc != null) {
final gate.Document md = markedDoc;
javax.swing.SwingUtilities.invokeLater(new Runnable() {
@Override
public void run() {
Factory.deleteResource(md);
}
});
}
}
// for loop through saved docs
sds.close();
} catch (java.net.MalformedURLException ex) {
throw (GateRuntimeException) new GateRuntimeException("CorpusBenchmark: " + ex.getMessage()).initCause(ex);
} catch (PersistenceException ex1) {
throw (GateRuntimeException) new GateRuntimeException("CorpusBenchmark: " + ex1.getMessage()).initCause(ex1);
} catch (ResourceInstantiationException ex2) {
throw (GateRuntimeException) new GateRuntimeException("CorpusBenchmark: " + ex2.getMessage()).initCause(ex2);
}
}
use of gate.DataStore in project gate-core by GateNLP.
the class SerialCorpusImpl method resourceDeleted.
/**
* Called by a datastore when a resource has been deleted
*/
@Override
public void resourceDeleted(DatastoreEvent evt) {
DataStore ds = (DataStore) evt.getSource();
// 1. check whether this datastore fired the event. If not, return.
if (!ds.equals(this.dataStore))
return;
Object docID = evt.getResourceID();
if (docID == null)
return;
if (DEBUG)
Out.prln("Resource deleted called for: " + docID);
// unloaded immediately
if (docID.equals(this.getLRPersistenceId())) {
Factory.deleteResource(this);
return;
}
// if
boolean isDirty = false;
// first
for (int i = 0; i < docDataList.size(); i++) {
DocumentData docData = docDataList.get(i);
// don't break the loop, because it might appear more than once
if (docID.equals(docData.getPersistentID())) {
if (evt.getResource() == null) {
// instead of calling remove() which tries to load the
// document
// remove it from the documents and docDataList
documentRemoved(docDataList.get(i).persistentID.toString());
docDataList.remove(i);
documents.remove(i);
isDirty = true;
i--;
continue;
}
remove(i);
isDirty = true;
}
// if
}
if (isDirty)
try {
this.dataStore.sync(this);
} catch (PersistenceException ex) {
throw new GateRuntimeException("SerialCorpusImpl: " + ex.getMessage());
} catch (SecurityException sex) {
throw new GateRuntimeException("SerialCorpusImpl: " + sex.getMessage());
}
}
Aggregations