use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class DataLoaderSettings method loadDataWithSettings.
/**
* Kevin's fast data reader
*
* @param file
* @return DataModel on success
*/
public DataModel loadDataWithSettings(File file) throws IOException {
DataModel dataModel = null;
Delimiter delimiter = getDelimiterType();
boolean hasHeader = isVarNamesFirstRow();
String commentMarker = getCommentMarker();
String missingValueMarker = getMissingValueMarker();
if (tabularRadioButton.isSelected()) {
TabularDataReader dataReader = null;
// Continuous, discrete, mixed
if (contRadioButton.isSelected()) {
dataReader = new ContinuousTabularDataFileReader(file, delimiter);
} else if (discRadioButton.isSelected()) {
dataReader = new VerticalDiscreteTabularDataReader(file, delimiter);
} else if (mixedRadioButton.isSelected()) {
dataReader = new MixedTabularDataFileReader(getMaxNumOfDiscCategories(), file, delimiter);
} else {
throw new UnsupportedOperationException("Unsupported data type!");
}
// Header in first row or not
dataReader.setHasHeader(hasHeader);
// Set comment marker
dataReader.setCommentMarker(commentMarker);
dataReader.setMissingValueMarker(missingValueMarker);
// Set the quote character
if (doubleQuoteRadioButton.isSelected()) {
dataReader.setQuoteCharacter('"');
}
if (singleQuoteRadioButton.isSelected()) {
dataReader.setQuoteCharacter('\'');
}
Dataset dataset;
// Handle case ID column based on different selections
if (idNoneRadioButton.isSelected()) {
// No column exclusion
dataset = dataReader.readInData();
} else if (idUnlabeledFirstColRadioButton.isSelected()) {
// Exclude the first column
dataset = dataReader.readInData(new int[] { 1 });
} else if (idLabeledColRadioButton.isSelected() && !idStringField.getText().isEmpty()) {
// Exclude the specified labled column
dataset = dataReader.readInData(new HashSet<>(Arrays.asList(new String[] { idStringField.getText() })));
} else {
throw new UnsupportedOperationException("Unexpected 'Case ID column to ignore' selection.");
}
// Box Dataset to DataModel
dataModel = DataConvertUtils.toDataModel(dataset);
} else if (covarianceRadioButton.isSelected()) {
// Covariance data can only be continuous
CovarianceDataReader dataReader = new LowerCovarianceDataReader(file, delimiter);
// Set comment marker
dataReader.setCommentMarker(commentMarker);
// Set the quote character
if (doubleQuoteRadioButton.isSelected()) {
dataReader.setQuoteCharacter('"');
}
if (singleQuoteRadioButton.isSelected()) {
dataReader.setQuoteCharacter('\'');
}
Dataset dataset = dataReader.readInData();
// Box Dataset to DataModel
dataModel = DataConvertUtils.toDataModel(dataset);
} else {
throw new UnsupportedOperationException("Unsupported selection of File Type!");
}
return dataModel;
}
use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class GdistanceApply method main.
public static void main(String... args) {
double xdist = 2.4;
double ydist = 2.4;
double zdist = 2;
long timestart = System.nanoTime();
System.out.println("Loading first graph");
Graph graph1 = GraphUtils.loadGraphTxt(new File("Motion_Corrected_Graphs/singlesub_motion_graph_025_04.txt"));
long timegraph1 = System.nanoTime();
// System.out.println(graph1);
System.out.println("Done loading first graph. Elapsed time: " + (timegraph1 - timestart) / 1000000000 + "s");
System.out.println("Loading second graph");
Graph graph2 = GraphUtils.loadGraphTxt(new File("Motion_Corrected_Graphs/singlesub_motion_graph_027_04.txt"));
long timegraph2 = System.nanoTime();
System.out.println("Done loading second graph. Elapsed time: " + (timegraph2 - timegraph1) / 1000000000 + "s");
// +++++++++ these steps are specifically for the motion corrected fMRI graphs ++++++++++++
graph1.removeNode(graph1.getNode("Motion_1"));
graph1.removeNode(graph1.getNode("Motion_2"));
graph1.removeNode(graph1.getNode("Motion_3"));
graph1.removeNode(graph1.getNode("Motion_4"));
graph1.removeNode(graph1.getNode("Motion_5"));
graph1.removeNode(graph1.getNode("Motion_6"));
graph2.removeNode(graph2.getNode("Motion_1"));
graph2.removeNode(graph2.getNode("Motion_2"));
graph2.removeNode(graph2.getNode("Motion_3"));
graph2.removeNode(graph2.getNode("Motion_4"));
graph2.removeNode(graph2.getNode("Motion_5"));
graph2.removeNode(graph2.getNode("Motion_6"));
// load the location map
String workingDirectory = System.getProperty("user.dir");
System.out.println(workingDirectory);
Path mapPath = Paths.get("coords.txt");
System.out.println(mapPath);
TabularDataReader dataReaderMap = new ContinuousTabularDataFileReader(mapPath.toFile(), Delimiter.COMMA);
try {
DataSet locationMap = (DataSet) DataConvertUtils.toDataModel(dataReaderMap.readInData());
long timegraph3 = System.nanoTime();
System.out.println("Done loading location map. Elapsed time: " + (timegraph3 - timegraph2) / 1000000000 + "s");
System.out.println("Running Gdistance");
Gdistance gdist = new Gdistance(locationMap, xdist, ydist, zdist);
List<Double> distance = gdist.distances(graph1, graph2);
System.out.println(distance);
System.out.println("Done running Distance. Elapsed time: " + (System.nanoTime() - timegraph3) / 1000000000 + "s");
System.out.println("Total elapsed time: " + (System.nanoTime() - timestart) / 1000000000 + "s");
PrintWriter writer = new PrintWriter("Gdistances.txt", "UTF-8");
writer.println(distance);
writer.close();
} catch (Exception IOException) {
IOException.printStackTrace();
}
}
use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class GdistanceTest method main.
public static void main(String... args) {
// first generate a couple random graphs
int numVars = 16;
int numEdges = 16;
List<Node> vars = new ArrayList<>();
for (int i = 0; i < numVars; i++) {
vars.add(new ContinuousVariable("X" + i));
}
Graph testdag1 = GraphUtils.randomGraphRandomForwardEdges(vars, 0, numEdges, 30, 15, 15, false, true);
Graph testdag2 = GraphUtils.randomGraphRandomForwardEdges(vars, 0, numEdges, 30, 15, 15, false, true);
// System.out.println(testdag1);
// load the location map
String workingDirectory = System.getProperty("user.dir");
System.out.println(workingDirectory);
Path mapPath = Paths.get("locationMap.txt");
System.out.println(mapPath);
TabularDataReader dataReaderMap = new ContinuousTabularDataFileReader(mapPath.toFile(), Delimiter.COMMA);
try {
DataSet locationMap = (DataSet) DataConvertUtils.toDataModel(dataReaderMap.readInData());
// System.out.println(locationMap);
// then compare their distance
double xdist = 2.4;
double ydist = 2.4;
double zdist = 2;
Gdistance gdist = new Gdistance(locationMap, xdist, ydist, zdist);
List<Double> output = gdist.distances(testdag1, testdag2);
System.out.println(output);
PrintWriter writer = new PrintWriter("Gdistances.txt", "UTF-8");
writer.println(output);
writer.close();
} catch (Exception IOException) {
IOException.printStackTrace();
}
}
use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class HsimEvalFromData method main.
public static void main(String[] args) {
long timestart = System.nanoTime();
System.out.println("Beginning Evaluation");
String nl = System.lineSeparator();
String output = "Simulation edu.cmu.tetrad.study output comparing Fsim and Hsim on predicting graph discovery accuracy" + nl;
int iterations = 100;
int vars = 20;
int cases = 500;
int edgeratio = 3;
List<Integer> hsimRepeat = Arrays.asList(40);
List<Integer> fsimRepeat = Arrays.asList(40);
List<PRAOerrors>[] fsimErrsByPars = new ArrayList[fsimRepeat.size()];
int whichFrepeat = 0;
for (int frepeat : fsimRepeat) {
fsimErrsByPars[whichFrepeat] = new ArrayList<PRAOerrors>();
whichFrepeat++;
}
List<PRAOerrors>[][] hsimErrsByPars = new ArrayList[1][hsimRepeat.size()];
// System.out.println(resimSize.size()+" "+hsimRepeat.size());
int whichHrepeat;
whichHrepeat = 0;
for (int hrepeat : hsimRepeat) {
// System.out.println(whichrsize+" "+whichHrepeat);
hsimErrsByPars[0][whichHrepeat] = new ArrayList<PRAOerrors>();
whichHrepeat++;
}
// !(*%(@!*^!($%!^ START ITERATING HERE !#$%(*$#@!^(*!$*%(!$#
try {
for (int iterate = 0; iterate < iterations; iterate++) {
System.out.println("iteration " + iterate);
// @#$%@$%^@$^@$^@%$%@$#^ LOADING THE DATA AND GRAPH @$#%%*#^##*^$#@%$
DataSet data1;
Graph graph1 = GraphUtils.loadGraphTxt(new File("graph/graph.1.txt"));
Dag odag = new Dag(graph1);
Set<String> eVars = new HashSet<String>();
eVars.add("MULT");
Path dataFile = Paths.get("data/data.1.txt");
TabularDataReader dataReader = new ContinuousTabularDataFileReader(dataFile.toFile(), Delimiter.TAB);
data1 = (DataSet) DataConvertUtils.toDataModel(dataReader.readInData(eVars));
vars = data1.getNumColumns();
cases = data1.getNumRows();
edgeratio = 3;
// !#@^$@&%^!#$!&@^ CALCULATING TARGET ERRORS $%$#@^@!%!#^$!%$#%
ICovarianceMatrix newcov = new CovarianceMatrixOnTheFly(data1);
SemBicScore oscore = new SemBicScore(newcov);
Fges ofgs = new Fges(oscore);
ofgs.setVerbose(false);
ofgs.setNumPatternsToStore(0);
// ***********This is the original FGS output on the data
Graph oFGSGraph = ofgs.search();
PRAOerrors oErrors = new PRAOerrors(HsimUtils.errorEval(oFGSGraph, odag), "target errors");
// **then step 1: full resim. iterate through the combinations of estimator parameters (just repeat num)
for (whichFrepeat = 0; whichFrepeat < fsimRepeat.size(); whichFrepeat++) {
ArrayList<PRAOerrors> errorsList = new ArrayList<PRAOerrors>();
for (int r = 0; r < fsimRepeat.get(whichFrepeat); r++) {
PatternToDag pickdag = new PatternToDag(oFGSGraph);
Graph fgsDag = pickdag.patternToDagMeek();
Dag fgsdag2 = new Dag(fgsDag);
// then fit an IM to this dag and the data. GeneralizedSemEstimator seems to bug out
// GeneralizedSemPm simSemPm = new GeneralizedSemPm(fgsdag2);
// GeneralizedSemEstimator gsemEstimator = new GeneralizedSemEstimator();
// GeneralizedSemIm fittedIM = gsemEstimator.estimate(simSemPm, oData);
SemPm simSemPm = new SemPm(fgsdag2);
// BayesPm simBayesPm = new BayesPm(fgsdag2, bayesPm);
SemEstimator simSemEstimator = new SemEstimator(data1, simSemPm);
SemIm fittedIM = simSemEstimator.estimate();
DataSet simData = fittedIM.simulateData(data1.getNumRows(), false);
// after making the full resim data (simData), run FGS on that
ICovarianceMatrix simcov = new CovarianceMatrixOnTheFly(simData);
SemBicScore simscore = new SemBicScore(simcov);
Fges simfgs = new Fges(simscore);
simfgs.setVerbose(false);
simfgs.setNumPatternsToStore(0);
Graph simGraphOut = simfgs.search();
PRAOerrors simErrors = new PRAOerrors(HsimUtils.errorEval(simGraphOut, fgsdag2), "Fsim errors " + r);
errorsList.add(simErrors);
}
PRAOerrors avErrors = new PRAOerrors(errorsList, "Average errors for Fsim at repeat=" + fsimRepeat.get(whichFrepeat));
// if (verbosity>3) System.out.println(avErrors.allToString());
// ****calculate the squared errors of prediction, store all these errors in a list
double FsimAR2 = (avErrors.getAdjRecall() - oErrors.getAdjRecall()) * (avErrors.getAdjRecall() - oErrors.getAdjRecall());
double FsimAP2 = (avErrors.getAdjPrecision() - oErrors.getAdjPrecision()) * (avErrors.getAdjPrecision() - oErrors.getAdjPrecision());
double FsimOR2 = (avErrors.getOrientRecall() - oErrors.getOrientRecall()) * (avErrors.getOrientRecall() - oErrors.getOrientRecall());
double FsimOP2 = (avErrors.getOrientPrecision() - oErrors.getOrientPrecision()) * (avErrors.getOrientPrecision() - oErrors.getOrientPrecision());
PRAOerrors Fsim2 = new PRAOerrors(new double[] { FsimAR2, FsimAP2, FsimOR2, FsimOP2 }, "squared errors for Fsim at repeat=" + fsimRepeat.get(whichFrepeat));
// add the fsim squared errors to the appropriate list
fsimErrsByPars[whichFrepeat].add(Fsim2);
}
// **then step 2: hybrid sim. iterate through combos of params (repeat num, resimsize)
for (whichHrepeat = 0; whichHrepeat < hsimRepeat.size(); whichHrepeat++) {
HsimRepeatAC study = new HsimRepeatAC(data1);
PRAOerrors HsimErrors = new PRAOerrors(study.run(1, hsimRepeat.get(whichHrepeat)), "Hsim errors" + "at rsize=" + 1 + " repeat=" + hsimRepeat.get(whichHrepeat));
// ****calculate the squared errors of prediction
double HsimAR2 = (HsimErrors.getAdjRecall() - oErrors.getAdjRecall()) * (HsimErrors.getAdjRecall() - oErrors.getAdjRecall());
double HsimAP2 = (HsimErrors.getAdjPrecision() - oErrors.getAdjPrecision()) * (HsimErrors.getAdjPrecision() - oErrors.getAdjPrecision());
double HsimOR2 = (HsimErrors.getOrientRecall() - oErrors.getOrientRecall()) * (HsimErrors.getOrientRecall() - oErrors.getOrientRecall());
double HsimOP2 = (HsimErrors.getOrientPrecision() - oErrors.getOrientPrecision()) * (HsimErrors.getOrientPrecision() - oErrors.getOrientPrecision());
PRAOerrors Hsim2 = new PRAOerrors(new double[] { HsimAR2, HsimAP2, HsimOR2, HsimOP2 }, "squared errors for Hsim, rsize=" + 1 + " repeat=" + hsimRepeat.get(whichHrepeat));
hsimErrsByPars[0][whichHrepeat].add(Hsim2);
}
}
// Average the squared errors for each set of fsim/hsim params across all iterations
PRAOerrors[] fMSE = new PRAOerrors[fsimRepeat.size()];
PRAOerrors[][] hMSE = new PRAOerrors[1][hsimRepeat.size()];
String[][] latexTableArray = new String[1 * hsimRepeat.size() + fsimRepeat.size()][5];
for (int j = 0; j < fMSE.length; j++) {
fMSE[j] = new PRAOerrors(fsimErrsByPars[j], "MSE for Fsim at vars=" + vars + " edgeratio=" + edgeratio + " cases=" + cases + " frepeat=" + fsimRepeat.get(j) + " iterations=" + iterations);
// if(verbosity>0){System.out.println(fMSE[j].allToString());}
output = output + fMSE[j].allToString() + nl;
latexTableArray[j] = prelimToPRAOtable(fMSE[j]);
}
for (int j = 0; j < hMSE.length; j++) {
for (int k = 0; k < hMSE[j].length; k++) {
hMSE[j][k] = new PRAOerrors(hsimErrsByPars[j][k], "MSE for Hsim at vars=" + vars + " edgeratio=" + edgeratio + " cases=" + cases + " rsize=" + 1 + " repeat=" + hsimRepeat.get(k) + " iterations=" + iterations);
// if(verbosity>0){System.out.println(hMSE[j][k].allToString());}
output = output + hMSE[j][k].allToString() + nl;
latexTableArray[fsimRepeat.size() + j * hMSE[j].length + k] = prelimToPRAOtable(hMSE[j][k]);
}
}
// record all the params, the base error values, and the fsim/hsim mean squared errors
String latexTable = HsimUtils.makeLatexTable(latexTableArray);
PrintWriter writer = new PrintWriter("latexTable.txt", "UTF-8");
writer.println(latexTable);
writer.close();
PrintWriter writer2 = new PrintWriter("HvsF-SimulationEvaluation.txt", "UTF-8");
writer2.println(output);
writer2.close();
long timestop = System.nanoTime();
System.out.println("Evaluation Concluded. Duration: " + (timestop - timestart) / 1000000000 + "s");
} catch (Exception IOException) {
IOException.printStackTrace();
}
}
use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class HsimRun method run.
public static void run(String readfilename, String filenameOut, char delimiter, String[] resimNodeNames, boolean verbose) {
// ===========read data from file=============
String workingDirectory = System.getProperty("user.dir");
System.out.println(workingDirectory);
try {
Path dataFile = Paths.get(readfilename);
TabularDataReader dataReader = new VerticalDiscreteTabularDataReader(dataFile.toFile(), DelimiterUtils.toDelimiter(delimiter));
DataSet dataSet = (DataSet) DataConvertUtils.toDataModel(dataReader.readInData());
System.out.println("cols: " + dataSet.getNumColumns() + " rows: " + dataSet.getNumRows());
// testing the read file
// DataWriter.writeRectangularData(dataSet, new FileWriter("dataOut2.txt"), '\t');
// apply Hsim to data, with whatever parameters
// ========first make the Dag for Hsim==========
// ICovarianceMatrix cov = new CovarianceMatrixOnTheFly(dataSet);
double penaltyDiscount = 2.0;
SemBicScore score = new SemBicScore(new CovarianceMatrixOnTheFly(dataSet));
score.setPenaltyDiscount(penaltyDiscount);
Fges fges = new Fges(score);
fges.setVerbose(false);
fges.setNumPatternsToStore(0);
// fges.setCorrErrorsAlpha(penaltyDiscount);
// fges.setOut(out);
// fges.setFaithfulnessAssumed(true);
// fges.setMaxIndegree(1);
// fges.setCycleBound(5);
Graph estGraph = fges.search();
System.out.println(estGraph);
Graph estPattern = new EdgeListGraphSingleConnections(estGraph);
PatternToDag patternToDag = new PatternToDag(estPattern);
Graph estGraphDAG = patternToDag.patternToDagMeek();
Dag estDAG = new Dag(estGraphDAG);
// ===========Identify the nodes to be resimulated===========
// estDAG.getNodes()
// need to populate simnodes with the nodes to be resimulated
// for now, I choose a center Node and add its neighbors
/* ===Commented out, but saved for future use=====
Node centerNode = estDAG.getNode("X3");
Set<Node> simnodes = new HashSet<Node>();
simnodes.add(centerNode);
simnodes.addAll(estDAG.getAdjacentNodes(centerNode));
*/
// ===test code, for user input specifying specific set of resim nodes====
// user needs to specify a list or array or something of node names
// use for loop through that collection, get each node from the names, add to the set
Set<Node> simnodes = new HashSet<>();
for (int i = 0; i < resimNodeNames.length; i++) {
Node thisNode = estDAG.getNode(resimNodeNames[i]);
simnodes.add(thisNode);
}
// ===========Apply the hybrid resimulation===============
Hsim hsim = new Hsim(estDAG, simnodes, dataSet);
DataSet newDataSet = hsim.hybridsimulate();
// write output to a new file
DataWriter.writeRectangularData(newDataSet, new FileWriter(filenameOut), delimiter);
// =======Run FGES on the output data, and compare it to the original learned graph
Path dataFileOut = Paths.get(filenameOut);
TabularDataReader dataReaderOut = new VerticalDiscreteTabularDataReader(dataFileOut.toFile(), DelimiterUtils.toDelimiter(delimiter));
DataSet dataSetOut = (DataSet) DataConvertUtils.toDataModel(dataReaderOut.readInData());
SemBicScore _score = new SemBicScore(new CovarianceMatrix(dataSetOut));
_score.setPenaltyDiscount(2.0);
Fges fgesOut = new Fges(_score);
fgesOut.setVerbose(false);
fgesOut.setNumPatternsToStore(0);
// fgesOut.setCorrErrorsAlpha(2.0);
// fgesOut.setOut(out);
// fgesOut.setFaithfulnessAssumed(true);
// fgesOut.setMaxIndegree(1);
// fgesOut.setCycleBound(5);
Graph estGraphOut = fgesOut.search();
System.out.println(estGraphOut);
SearchGraphUtils.graphComparison(estGraphOut, estGraph, System.out);
} catch (Exception IOException) {
IOException.printStackTrace();
}
}
Aggregations