use of org.apache.parquet.column.page.PageReadStore in project ignite by apache.
the class SparkModelParser method loadKMeansModel.
/**
* Load K-Means model.
*
* @param pathToMdl Path to model.
* @param learningEnvironment learningEnvironment
*/
private static Model loadKMeansModel(String pathToMdl, LearningEnvironment learningEnvironment) {
Vector[] centers = null;
try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
PageReadStore pages;
final MessageType schema = r.getFooter().getFileMetaData().getSchema();
final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
while (null != (pages = r.readNextRowGroup())) {
final int rows = (int) pages.getRowCount();
final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
centers = new DenseVector[rows];
for (int i = 0; i < rows; i++) {
final SimpleGroup g = (SimpleGroup) recordReader.read();
// final int clusterIdx = g.getInteger(0, 0);
Group clusterCenterCoeff = g.getGroup(1, 0).getGroup(3, 0);
final int amountOfCoefficients = clusterCenterCoeff.getFieldRepetitionCount(0);
centers[i] = new DenseVector(amountOfCoefficients);
for (int j = 0; j < amountOfCoefficients; j++) {
double coefficient = clusterCenterCoeff.getGroup(0, j).getDouble(0, 0);
centers[i].set(j, coefficient);
}
}
}
} catch (IOException e) {
String msg = "Error reading parquet file: " + e.getMessage();
learningEnvironment.logger().log(MLLogger.VerboseLevel.HIGH, msg);
e.printStackTrace();
}
return new KMeansModel(centers, new EuclideanDistance());
}
use of org.apache.parquet.column.page.PageReadStore in project ignite by apache.
the class SparkModelParser method loadLinearSVMModel.
/**
* Load SVM model.
*
* @param pathToMdl Path to model.
* @param learningEnvironment Learning environment.
*/
private static Model loadLinearSVMModel(String pathToMdl, LearningEnvironment learningEnvironment) {
Vector coefficients = null;
double interceptor = 0;
try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
PageReadStore pages;
final MessageType schema = r.getFooter().getFileMetaData().getSchema();
final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
while (null != (pages = r.readNextRowGroup())) {
final long rows = pages.getRowCount();
final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
for (int i = 0; i < rows; i++) {
final SimpleGroup g = (SimpleGroup) recordReader.read();
interceptor = readSVMInterceptor(g);
coefficients = readSVMCoefficients(g);
}
}
} catch (IOException e) {
String msg = "Error reading parquet file: " + e.getMessage();
learningEnvironment.logger().log(MLLogger.VerboseLevel.HIGH, msg);
e.printStackTrace();
}
return new SVMLinearClassificationModel(coefficients, interceptor);
}
use of org.apache.parquet.column.page.PageReadStore in project ignite by apache.
the class SparkModelParser method loadDecisionTreeModel.
/**
* Load Decision Tree model.
*
* @param pathToMdl Path to model.
* @param learningEnvironment Learning environment.
*/
private static Model loadDecisionTreeModel(String pathToMdl, LearningEnvironment learningEnvironment) {
try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
PageReadStore pages;
final MessageType schema = r.getFooter().getFileMetaData().getSchema();
final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
final Map<Integer, NodeData> nodes = new TreeMap<>();
while (null != (pages = r.readNextRowGroup())) {
final long rows = pages.getRowCount();
final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
for (int i = 0; i < rows; i++) {
final SimpleGroup g = (SimpleGroup) recordReader.read();
NodeData nodeData = extractNodeDataFromParquetRow(g);
nodes.put(nodeData.id, nodeData);
}
}
return buildDecisionTreeModel(nodes);
} catch (IOException e) {
String msg = "Error reading parquet file: " + e.getMessage();
learningEnvironment.logger().log(MLLogger.VerboseLevel.HIGH, msg);
e.printStackTrace();
}
return null;
}
use of org.apache.parquet.column.page.PageReadStore in project ignite by apache.
the class SparkModelParser method loadLogRegModel.
/**
* Load logistic regression model.
*
* @param pathToMdl Path to model.
* @param learningEnvironment Learning environment.
*/
private static Model loadLogRegModel(String pathToMdl, LearningEnvironment learningEnvironment) {
Vector coefficients = null;
double interceptor = 0;
try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
PageReadStore pages;
final MessageType schema = r.getFooter().getFileMetaData().getSchema();
final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
while (null != (pages = r.readNextRowGroup())) {
final long rows = pages.getRowCount();
final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
for (int i = 0; i < rows; i++) {
final SimpleGroup g = (SimpleGroup) recordReader.read();
interceptor = readInterceptor(g);
coefficients = readCoefficients(g);
}
}
} catch (IOException e) {
String msg = "Error reading parquet file: " + e.getMessage();
learningEnvironment.logger().log(MLLogger.VerboseLevel.HIGH, msg);
e.printStackTrace();
}
return new LogisticRegressionModel(coefficients, interceptor);
}
use of org.apache.parquet.column.page.PageReadStore in project ignite by apache.
the class SparkModelParser method loadLinRegModel.
/**
* Load linear regression model.
*
* @param pathToMdl Path to model.
* @param learningEnvironment Learning environment.
*/
private static Model loadLinRegModel(String pathToMdl, LearningEnvironment learningEnvironment) {
Vector coefficients = null;
double interceptor = 0;
try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
PageReadStore pages;
final MessageType schema = r.getFooter().getFileMetaData().getSchema();
final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
while (null != (pages = r.readNextRowGroup())) {
final long rows = pages.getRowCount();
final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
for (int i = 0; i < rows; i++) {
final SimpleGroup g = (SimpleGroup) recordReader.read();
interceptor = readLinRegInterceptor(g);
coefficients = readLinRegCoefficients(g);
}
}
} catch (IOException e) {
String msg = "Error reading parquet file: " + e.getMessage();
learningEnvironment.logger().log(MLLogger.VerboseLevel.HIGH, msg);
e.printStackTrace();
}
return new LinearRegressionModel(coefficients, interceptor);
}
Aggregations