use of org.tribuo.Example in project ml-commons by opensearch-project.
the class TribuoUtil method generateDatasetWithTarget.
/**
* Generate tribuo dataset from data frame with target.
* @param dataFrame features data
* @param outputFactory the tribuo output factory
* @param desc description for tribuo provenance
* @param outputType the tribuo output type
* @param target target name
* @return tribuo dataset
*/
public static <T extends Output<T>> MutableDataset<T> generateDatasetWithTarget(DataFrame dataFrame, OutputFactory<T> outputFactory, String desc, TribuoOutputType outputType, String target) {
if (StringUtils.isEmpty(target)) {
throw new IllegalArgumentException("Empty target when generating dataset from data frame.");
}
List<Example<T>> dataset = new ArrayList<>();
Tuple<String[], double[][]> featureNamesValues = transformDataFrame(dataFrame);
int targetIndex = -1;
for (int i = 0; i < featureNamesValues.v1().length; ++i) {
if (featureNamesValues.v1()[i].equals(target)) {
targetIndex = i;
break;
}
}
if (targetIndex == -1) {
throw new IllegalArgumentException("No matched target when generating dataset from data frame.");
}
ArrayExample<T> example;
final int finalTargetIndex = targetIndex;
String[] featureNames = IntStream.range(0, featureNamesValues.v1().length).filter(e -> e != finalTargetIndex).mapToObj(e -> featureNamesValues.v1()[e]).toArray(String[]::new);
for (int i = 0; i < dataFrame.size(); ++i) {
switch(outputType) {
case REGRESSOR:
final int finalI = i;
double targetValue = featureNamesValues.v2()[finalI][finalTargetIndex];
double[] featureValues = IntStream.range(0, featureNamesValues.v2()[i].length).filter(e -> e != finalTargetIndex).mapToDouble(e -> featureNamesValues.v2()[finalI][e]).toArray();
example = new ArrayExample<>((T) new Regressor(target, targetValue), featureNames, featureValues);
break;
default:
throw new IllegalArgumentException("unknown type:" + outputType);
}
dataset.add(example);
}
SimpleDataSourceProvenance provenance = new SimpleDataSourceProvenance(desc, outputFactory);
return new MutableDataset<>(new ListDataSource<>(dataset, outputFactory, provenance));
}
use of org.tribuo.Example in project ml-commons by opensearch-project.
the class TribuoUtil method generateDataset.
/**
* Generate tribuo dataset from data frame.
* @param dataFrame features data
* @param outputFactory the tribuo output factory
* @param desc description for tribuo provenance
* @param outputType the tribuo output type
* @return tribuo dataset
*/
public static <T extends Output<T>> MutableDataset<T> generateDataset(DataFrame dataFrame, OutputFactory<T> outputFactory, String desc, TribuoOutputType outputType) {
List<Example<T>> dataset = new ArrayList<>();
Tuple<String[], double[][]> featureNamesValues = transformDataFrame(dataFrame);
ArrayExample<T> example;
for (int i = 0; i < dataFrame.size(); ++i) {
switch(outputType) {
case CLUSTERID:
example = new ArrayExample<>((T) new ClusterID(ClusterID.UNASSIGNED), featureNamesValues.v1(), featureNamesValues.v2()[i]);
break;
case REGRESSOR:
// Create single dimension tribuo regressor with name DIM-0 and value double NaN.
example = new ArrayExample<>((T) new Regressor("DIM-0", Double.NaN), featureNamesValues.v1(), featureNamesValues.v2()[i]);
break;
case ANOMALY_DETECTION_LIBSVM:
// Why we set default event type as EXPECTED(non-anomalous)
// 1. For training data, Tribuo LibSVMAnomalyTrainer only supports EXPECTED events at training time.
// 2. For prediction data, we treat the data as non-anomalous by default as Tribuo lib don't accept UNKNOWN type.
Event.EventType defaultEventType = Event.EventType.EXPECTED;
// TODO: support anomaly labels to evaluate prediction result
example = new ArrayExample<>((T) new Event(defaultEventType), featureNamesValues.v1(), featureNamesValues.v2()[i]);
break;
default:
throw new IllegalArgumentException("unknown type:" + outputType);
}
dataset.add(example);
}
SimpleDataSourceProvenance provenance = new SimpleDataSourceProvenance(desc, outputFactory);
return new MutableDataset<>(new ListDataSource<>(dataset, outputFactory, provenance));
}
use of org.tribuo.Example in project ml-commons by opensearch-project.
the class AnomalyDetectionLibSVMTest method constructDataFrame.
private DataFrame constructDataFrame(Dataset<Event> data, boolean training, List<Event.EventType> labels) {
Iterator<Example<Event>> iterator = data.iterator();
List<ColumnMeta> columns = null;
DataFrame dataFrame = null;
while (iterator.hasNext()) {
Example<Event> example = iterator.next();
if (columns == null) {
columns = new ArrayList<>();
List<ColumnValue> columnValues = new ArrayList<>();
for (Feature feature : example) {
columns.add(new ColumnMeta(feature.getName(), ColumnType.DOUBLE));
columnValues.add(new DoubleValue(feature.getValue()));
}
ColumnMeta[] columnMetas = columns.toArray(new ColumnMeta[columns.size()]);
dataFrame = new DefaultDataFrame(columnMetas);
addRow(columnValues, training, example, dataFrame, labels);
} else {
List<ColumnValue> columnValues = new ArrayList<>();
for (Feature feature : example) {
columnValues.add(new DoubleValue(feature.getValue()));
}
addRow(columnValues, training, example, dataFrame, labels);
}
}
return dataFrame;
}
use of org.tribuo.Example in project ml-commons by opensearch-project.
the class TribuoUtilTest method generateDatasetWithTarget.
@SuppressWarnings("unchecked")
@Test
public void generateDatasetWithTarget() {
MutableDataset<Regressor> dataset = TribuoUtil.generateDatasetWithTarget(dataFrame, new RegressionFactory(), "test", TribuoOutputType.REGRESSOR, "f2");
List<Example<Regressor>> examples = dataset.getData();
Assert.assertEquals(rawData.length, examples.size());
for (int i = 0; i < rawData.length; ++i) {
ArrayExample arrayExample = (ArrayExample) examples.get(i);
Iterator<Feature> iterator = arrayExample.iterator();
int idx = 1;
while (iterator.hasNext()) {
Feature feature = iterator.next();
Assert.assertEquals("f" + idx, feature.getName());
Assert.assertEquals(i + idx / 10.0, feature.getValue(), 0.01);
++idx;
}
}
}
use of org.tribuo.Example in project ml-commons by opensearch-project.
the class TribuoUtilTest method generateDataset.
@SuppressWarnings("unchecked")
@Test
public void generateDataset() {
MutableDataset<ClusterID> dataset = TribuoUtil.generateDataset(dataFrame, new ClusteringFactory(), "test", TribuoOutputType.CLUSTERID);
List<Example<ClusterID>> examples = dataset.getData();
Assert.assertEquals(rawData.length, examples.size());
for (int i = 0; i < rawData.length; ++i) {
ArrayExample arrayExample = (ArrayExample) examples.get(i);
Iterator<Feature> iterator = arrayExample.iterator();
int idx = 1;
while (iterator.hasNext()) {
Feature feature = iterator.next();
Assert.assertEquals("f" + idx, feature.getName());
Assert.assertEquals(i + idx / 10.0, feature.getValue(), 0.01);
++idx;
}
}
}
Aggregations