use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class Comparison2 method compare.
/**
* Simulates data from model parameterizing the given DAG, and runs the
* algorithm on that data, printing out error statistics.
*/
public static ComparisonResult compare(ComparisonParameters params) {
DataSet dataSet = null;
Graph trueDag = null;
IndependenceTest test = null;
Score score = null;
ComparisonResult result = new ComparisonResult(params);
if (params.isDataFromFile()) {
/**
* Set path to the data directory *
*/
String path = "/Users/dmalinsky/Documents/research/data/danexamples";
File dir = new File(path);
File[] files = dir.listFiles();
if (files == null) {
throw new NullPointerException("No files in " + path);
}
for (File file : files) {
if (file.getName().startsWith("graph") && file.getName().contains(String.valueOf(params.getGraphNum())) && file.getName().endsWith(".g.txt")) {
params.setGraphFile(file.getName());
trueDag = GraphUtils.loadGraphTxt(file);
break;
}
}
String trialGraph = String.valueOf(params.getGraphNum()).concat("-").concat(String.valueOf(params.getTrial())).concat(".dat.txt");
for (File file : files) {
if (file.getName().startsWith("graph") && file.getName().endsWith(trialGraph)) {
Path dataFile = Paths.get(path.concat("/").concat(file.getName()));
Delimiter delimiter = Delimiter.TAB;
if (params.getDataType() == ComparisonParameters.DataType.Continuous) {
try {
TabularDataReader dataReader = new ContinuousTabularDataFileReader(dataFile.toFile(), delimiter);
dataSet = (DataSet) DataConvertUtils.toDataModel(dataReader.readInData());
} catch (IOException e) {
e.printStackTrace();
}
params.setDataFile(file.getName());
break;
} else {
try {
TabularDataReader dataReader = new VerticalDiscreteTabularDataReader(dataFile.toFile(), delimiter);
dataSet = (DataSet) DataConvertUtils.toDataModel(dataReader.readInData());
} catch (IOException e) {
e.printStackTrace();
}
params.setDataFile(file.getName());
break;
}
}
}
System.out.println("current graph file = " + params.getGraphFile());
System.out.println("current data set file = " + params.getDataFile());
}
if (params.isNoData()) {
List<Node> nodes = new ArrayList<>();
for (int i = 0; i < params.getNumVars(); i++) {
nodes.add(new ContinuousVariable("X" + (i + 1)));
}
trueDag = GraphUtils.randomGraphRandomForwardEdges(nodes, 0, params.getNumEdges(), 10, 10, 10, false, true);
/**
* added 5.25.16 for tsFCI *
*/
if (params.getAlgorithm() == ComparisonParameters.Algorithm.TsFCI) {
trueDag = GraphUtils.randomGraphRandomForwardEdges(nodes, 0, params.getNumEdges(), 10, 10, 10, false, true);
trueDag = TimeSeriesUtils.graphToLagGraph(trueDag, 2);
System.out.println("Creating Time Lag Graph : " + trueDag);
}
/**
* ************************
*/
test = new IndTestDSep(trueDag);
score = new GraphScore(trueDag);
if (params.getAlgorithm() == null) {
throw new IllegalArgumentException("Algorithm not set.");
}
long time1 = System.currentTimeMillis();
if (params.getAlgorithm() == ComparisonParameters.Algorithm.PC) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
Pc search = new Pc(test);
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.CPC) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
Cpc search = new Cpc(test);
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.PCLocal) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
PcLocal search = new PcLocal(test);
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.PCStableMax) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
PcStableMax search = new PcStableMax(test);
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.FGES) {
if (score == null) {
throw new IllegalArgumentException("Score not set.");
}
Fges search = new Fges(score);
// search.setFaithfulnessAssumed(params.isOneEdgeFaithfulnessAssumed());
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.FCI) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
Fci search = new Fci(test);
result.setResultGraph(search.search());
result.setCorrectResult(new DagToPag(trueDag).convert());
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.GFCI) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
GFci search = new GFci(test, score);
result.setResultGraph(search.search());
result.setCorrectResult(new DagToPag(trueDag).convert());
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.TsFCI) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
TsFci search = new TsFci(test);
IKnowledge knowledge = getKnowledge(trueDag);
search.setKnowledge(knowledge);
result.setResultGraph(search.search());
result.setCorrectResult(new TsDagToPag(trueDag).convert());
System.out.println("Correct result for trial = " + result.getCorrectResult());
System.out.println("Search result for trial = " + result.getResultGraph());
} else {
throw new IllegalArgumentException("Unrecognized algorithm.");
}
long time2 = System.currentTimeMillis();
long elapsed = time2 - time1;
result.setElapsed(elapsed);
result.setTrueDag(trueDag);
return result;
} else if (params.getDataFile() != null) {
// dataSet = loadDataFile(params.getDataFile());
System.out.println("Using data from file... ");
if (params.getGraphFile() == null) {
throw new IllegalArgumentException("True graph file not set.");
} else {
System.out.println("Using graph from file... ");
// trueDag = GraphUtils.loadGraph(File params.getGraphFile());
}
} else {
if (params.getNumVars() == -1) {
throw new IllegalArgumentException("Number of variables not set.");
}
if (params.getNumEdges() == -1) {
throw new IllegalArgumentException("Number of edges not set.");
}
if (params.getDataType() == ComparisonParameters.DataType.Continuous) {
List<Node> nodes = new ArrayList<>();
for (int i = 0; i < params.getNumVars(); i++) {
nodes.add(new ContinuousVariable("X" + (i + 1)));
}
trueDag = GraphUtils.randomGraphRandomForwardEdges(nodes, 0, params.getNumEdges(), 10, 10, 10, false, true);
/**
* added 6.08.16 for tsFCI *
*/
if (params.getAlgorithm() == ComparisonParameters.Algorithm.TsFCI) {
trueDag = GraphUtils.randomGraphRandomForwardEdges(nodes, 0, params.getNumEdges(), 10, 10, 10, false, true);
trueDag = TimeSeriesUtils.graphToLagGraph(trueDag, 2);
System.out.println("Creating Time Lag Graph : " + trueDag);
}
if (params.getDataType() == null) {
throw new IllegalArgumentException("Data type not set or inferred.");
}
if (params.getSampleSize() == -1) {
throw new IllegalArgumentException("Sample size not set.");
}
LargeScaleSimulation sim = new LargeScaleSimulation(trueDag);
/**
* added 6.08.16 for tsFCI *
*/
if (params.getAlgorithm() == ComparisonParameters.Algorithm.TsFCI) {
sim.setCoefRange(0.20, 0.50);
}
/**
* added 6.08.16 for tsFCI *
*/
if (params.getAlgorithm() == ComparisonParameters.Algorithm.TsFCI) {
// // System.out.println("Coefs matrix : " + sim.getCoefs());
// System.out.println(MatrixUtils.toString(sim.getCoefficientMatrix()));
// // System.out.println("dim = " + sim.getCoefs()[1][1]);
// boolean isStableTetradMatrix = allEigenvaluesAreSmallerThanOneInModulus(new TetradMatrix(sim.getCoefficientMatrix()));
// //this TetradMatrix needs to be the matrix of coefficients from the SEM!
// if (!isStableTetradMatrix) {
// System.out.println("%%%%%%%%%% WARNING %%%%%%%%% not a stable set of eigenvalues for data generation");
// System.out.println("Skipping this attempt!");
// sim.setCoefRange(0.2, 0.5);
// dataSet = sim.simulateDataAcyclic(params.getSampleSize());
// }
//
// /***************************/
boolean isStableTetradMatrix;
int attempt = 1;
int tierSize = params.getNumVars();
int[] sub = new int[tierSize];
int[] sub2 = new int[tierSize];
for (int i = 0; i < tierSize; i++) {
sub[i] = i;
sub2[i] = tierSize + i;
}
do {
dataSet = sim.simulateDataFisher(params.getSampleSize());
// System.out.println("Variable Nodes : " + sim.getVariableNodes());
// System.out.println(MatrixUtils.toString(sim.getCoefficientMatrix()));
TetradMatrix coefMat = new TetradMatrix(sim.getCoefficientMatrix());
TetradMatrix B = coefMat.getSelection(sub, sub);
TetradMatrix Gamma1 = coefMat.getSelection(sub2, sub);
TetradMatrix Gamma0 = TetradMatrix.identity(tierSize).minus(B);
TetradMatrix A1 = Gamma0.inverse().times(Gamma1);
// TetradMatrix B2 = coefMat.getSelection(sub2, sub2);
// System.out.println("B matrix : " + B);
// System.out.println("B2 matrix : " + B2);
// System.out.println("Gamma1 matrix : " + Gamma1);
// isStableTetradMatrix = allEigenvaluesAreSmallerThanOneInModulus(new TetradMatrix(sim.getCoefficientMatrix()));
isStableTetradMatrix = TimeSeriesUtils.allEigenvaluesAreSmallerThanOneInModulus(A1);
System.out.println("isStableTetradMatrix? : " + isStableTetradMatrix);
attempt++;
} while ((!isStableTetradMatrix) && attempt <= 5);
if (!isStableTetradMatrix) {
System.out.println("%%%%%%%%%% WARNING %%%%%%%% not a stable coefficient matrix, forcing coefs to [0.15,0.3]");
System.out.println("Made " + (attempt - 1) + " attempts to get stable matrix.");
sim.setCoefRange(0.15, 0.3);
dataSet = sim.simulateDataFisher(params.getSampleSize());
} else {
System.out.println("Coefficient matrix is stable.");
}
}
} else if (params.getDataType() == ComparisonParameters.DataType.Discrete) {
List<Node> nodes = new ArrayList<>();
for (int i = 0; i < params.getNumVars(); i++) {
nodes.add(new DiscreteVariable("X" + (i + 1), 3));
}
trueDag = GraphUtils.randomGraphRandomForwardEdges(nodes, 0, params.getNumEdges(), 10, 10, 10, false, true);
if (params.getDataType() == null) {
throw new IllegalArgumentException("Data type not set or inferred.");
}
if (params.getSampleSize() == -1) {
throw new IllegalArgumentException("Sample size not set.");
}
int[] tiers = new int[nodes.size()];
for (int i = 0; i < nodes.size(); i++) {
tiers[i] = i;
}
BayesPm pm = new BayesPm(trueDag, 3, 3);
MlBayesIm im = new MlBayesIm(pm, MlBayesIm.RANDOM);
dataSet = im.simulateData(params.getSampleSize(), false, tiers);
} else {
throw new IllegalArgumentException("Unrecognized data type.");
}
if (dataSet == null) {
throw new IllegalArgumentException("No data set.");
}
}
if (params.getIndependenceTest() == ComparisonParameters.IndependenceTestType.FisherZ) {
if (params.getDataType() != null && params.getDataType() != ComparisonParameters.DataType.Continuous) {
throw new IllegalArgumentException("Data type previously set to something other than continuous.");
}
if (Double.isNaN(params.getAlpha())) {
throw new IllegalArgumentException("Alpha not set.");
}
test = new IndTestFisherZ(dataSet, params.getAlpha());
params.setDataType(ComparisonParameters.DataType.Continuous);
} else if (params.getIndependenceTest() == ComparisonParameters.IndependenceTestType.ChiSquare) {
if (params.getDataType() != null && params.getDataType() != ComparisonParameters.DataType.Discrete) {
throw new IllegalArgumentException("Data type previously set to something other than discrete.");
}
if (Double.isNaN(params.getAlpha())) {
throw new IllegalArgumentException("Alpha not set.");
}
test = new IndTestChiSquare(dataSet, params.getAlpha());
params.setDataType(ComparisonParameters.DataType.Discrete);
}
if (params.getScore() == ScoreType.SemBic) {
if (params.getDataType() != null && params.getDataType() != ComparisonParameters.DataType.Continuous) {
throw new IllegalArgumentException("Data type previously set to something other than continuous.");
}
if (Double.isNaN(params.getPenaltyDiscount())) {
throw new IllegalArgumentException("Penalty discount not set.");
}
SemBicScore semBicScore = new SemBicScore(new CovarianceMatrixOnTheFly(dataSet));
semBicScore.setPenaltyDiscount(params.getPenaltyDiscount());
score = semBicScore;
params.setDataType(ComparisonParameters.DataType.Continuous);
} else if (params.getScore() == ScoreType.BDeu) {
if (params.getDataType() != null && params.getDataType() != ComparisonParameters.DataType.Discrete) {
throw new IllegalArgumentException("Data type previously set to something other than discrete.");
}
if (Double.isNaN(params.getSamplePrior())) {
throw new IllegalArgumentException("Sample prior not set.");
}
if (Double.isNaN(params.getStructurePrior())) {
throw new IllegalArgumentException("Structure prior not set.");
}
score = new BDeuScore(dataSet);
((BDeuScore) score).setSamplePrior(params.getSamplePrior());
((BDeuScore) score).setStructurePrior(params.getStructurePrior());
params.setDataType(ComparisonParameters.DataType.Discrete);
params.setDataType(ComparisonParameters.DataType.Discrete);
}
if (params.getAlgorithm() == null) {
throw new IllegalArgumentException("Algorithm not set.");
}
long time1 = System.currentTimeMillis();
if (params.getAlgorithm() == ComparisonParameters.Algorithm.PC) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
Pc search = new Pc(test);
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.CPC) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
Cpc search = new Cpc(test);
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.PCLocal) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
PcLocal search = new PcLocal(test);
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.PCStableMax) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
PcStableMax search = new PcStableMax(test);
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.FGES) {
if (score == null) {
throw new IllegalArgumentException("Score not set.");
}
Fges search = new Fges(score);
// search.setFaithfulnessAssumed(params.isOneEdgeFaithfulnessAssumed());
result.setResultGraph(search.search());
result.setCorrectResult(SearchGraphUtils.patternForDag(new EdgeListGraph(trueDag)));
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.FCI) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
Fci search = new Fci(test);
result.setResultGraph(search.search());
result.setCorrectResult(new DagToPag(trueDag).convert());
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.GFCI) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
GFci search = new GFci(test, score);
result.setResultGraph(search.search());
result.setCorrectResult(new DagToPag(trueDag).convert());
} else if (params.getAlgorithm() == ComparisonParameters.Algorithm.TsFCI) {
if (test == null) {
throw new IllegalArgumentException("Test not set.");
}
TsFci search = new TsFci(test);
IKnowledge knowledge = getKnowledge(trueDag);
search.setKnowledge(knowledge);
result.setResultGraph(search.search());
result.setCorrectResult(new TsDagToPag(trueDag).convert());
} else {
throw new IllegalArgumentException("Unrecognized algorithm.");
}
long time2 = System.currentTimeMillis();
long elapsed = time2 - time1;
result.setElapsed(elapsed);
result.setTrueDag(trueDag);
return result;
}
use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class GdistanceRandomApply method main.
public static void main(String... args) {
// thresholds are the barriers between histogram buckets.
double[] thresholds;
thresholds = new double[5];
thresholds[0] = 0;
thresholds[1] = 2;
thresholds[2] = 4;
thresholds[3] = 6;
thresholds[4] = 8;
// load the location map
String workingDirectory = System.getProperty("user.dir");
System.out.println(workingDirectory);
Path mapPath = Paths.get("erich_coordinates.txt");
System.out.println(mapPath);
TabularDataReader dataReaderMap = new ContinuousTabularDataFileReader(mapPath.toFile(), Delimiter.COMMA);
try {
DataSet locationMap = (DataSet) DataConvertUtils.toDataModel(dataReaderMap.readInData());
System.out.println("locationMap loaded");
GdistanceRandom simRandGdistances = new GdistanceRandom(locationMap);
System.out.println("GdistanceRandom constructed");
simRandGdistances.setNumEdges1(300);
simRandGdistances.setNumEdges2(300);
simRandGdistances.setVerbose(false);
System.out.println("Edge parameters set, starting simulations");
List<List<Double>> GdistanceLists = simRandGdistances.randomSimulation(2);
System.out.println("Simulations done, calculating histograms");
for (List<Double> gdist : GdistanceLists) {
double[] histogram = GdistanceUtils.histogram(gdist, thresholds);
// making the string to print out histogram values
String histString = " ";
for (int i = 0; i < Array.getLength(histogram); i++) {
histString = histString + " " + histogram[i];
}
System.out.println(histString);
}
} catch (Exception IOException) {
IOException.printStackTrace();
}
}
use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class TestGFci method testDiscreteData.
@Test
public void testDiscreteData() throws IOException {
double alpha = 0.05;
char delimiter = '\t';
Path dataFile = Paths.get("./src/test/resources/sim_discrete_data_20vars_100cases.txt");
TabularDataReader dataReader = new VerticalDiscreteTabularDataReader(dataFile.toFile(), DelimiterUtils.toDelimiter(delimiter));
DataSet dataSet = (DataSet) DataConvertUtils.toDataModel(dataReader.readInData());
IndependenceTest indTest = new IndTestChiSquare(dataSet, alpha);
BDeuScore score = new BDeuScore(dataSet);
score.setStructurePrior(1.0);
score.setSamplePrior(1.0);
GFci gFci = new GFci(indTest, score);
gFci.setFaithfulnessAssumed(true);
gFci.setMaxDegree(-1);
gFci.setMaxPathLength(-1);
gFci.setCompleteRuleSetUsed(false);
gFci.setVerbose(true);
long start = System.currentTimeMillis();
gFci.search();
long stop = System.currentTimeMillis();
System.out.println("Elapsed " + (stop - start) + " ms");
}
use of edu.pitt.dbmi.data.reader.tabular.TabularDataReader in project tetrad by cmu-phil.
the class TestDM method readAndSearchData.
// Reads in data and runs search. Note: Assumes variable names are of the form X0, X1, etc.
// Both input and output integer arrays are the indexes of their respective variables.
public DMSearch readAndSearchData(String fileLocation, int[] inputs, int[] outputs, boolean useGES, int[] trueInputs) {
File file = new File(fileLocation);
DataSet data = null;
try {
TabularDataReader dataReader = new ContinuousTabularDataFileReader(file, Delimiter.SPACE);
data = (DataSet) DataConvertUtils.toDataModel(dataReader.readInData());
} catch (IOException e) {
print("Failed to read in data.");
e.printStackTrace();
}
print("Read Data");
DMSearch search = new DMSearch();
search.setInputs(inputs);
search.setOutputs(outputs);
if (useGES == false) {
search.setAlphaPC(.05);
search.setUseFges(useGES);
search.setData(data);
search.setTrueInputs(trueInputs);
search.search();
} else {
search.setData(data);
search.setTrueInputs(trueInputs);
search.search();
// search.search(data, trueInputs);
}
return (search);
}
Aggregations