Examples with TrainingDataset io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset used on opensource projects

Example 16 with TrainingDataset

use of io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset in project hopsworks by logicalclocks.

the class TrainingDatasetControllerTest method testconvertToFilterEntities_rightOnlyFiler.

@Test
public void testconvertToFilterEntities_rightOnlyFiler() throws Exception {
    // fg.feature > 2
    // "fg.feature > 2" stores in the right-hand side
    TrainingDataset trainingDataset = new TrainingDataset();
    Feature f1 = new Feature("test_f", "fg0");
    FilterLogic head = new FilterLogic();
    head.setType(SINGLE);
    Filter right = new Filter(f1, GREATER_THAN, "2");
    head.setRightFilter(right);
    List<TrainingDatasetFilter> actual = target.convertToFilterEntities(head, trainingDataset, "L");
    List<TrainingDatasetFilter> expected = new ArrayList<>();
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "2"), SINGLE, "L"));
    Assert.assertEquals(expected.size(), actual.size());
    Assert.assertTrue(expected.containsAll(actual));
    Assert.assertTrue(actual.containsAll(expected));
}

Example 17 with TrainingDataset

use of io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset in project hopsworks by logicalclocks.

the class TrainingDatasetControllerTest method testconvertToFilterEntities_leftFilterRightLogic.

@Test
public void testconvertToFilterEntities_leftFilterRightLogic() throws Exception {
    // fg.feature > 1 and (fg.feature > 2 OR fg.feature > 3)
    // "fg.feature > 1" stores as filter
    TrainingDataset trainingDataset = new TrainingDataset();
    Feature f1 = new Feature("test_f", "fg0");
    FilterLogic head = new FilterLogic();
    head.setType(AND);
    Filter left = new Filter(f1, GREATER_THAN, "1");
    head.setLeftFilter(left);
    FilterLogic right = new FilterLogic();
    right.setType(OR);
    Filter right_left = new Filter(f1, GREATER_THAN, "2");
    Filter right_right = new Filter(f1, GREATER_THAN, "3");
    right.setLeftFilter(right_left);
    right.setRightFilter(right_right);
    head.setRightLogic(right);
    List<TrainingDatasetFilter> actual = target.convertToFilterEntities(head, trainingDataset, "L");
    List<TrainingDatasetFilter> expected = new ArrayList<>();
    expected.add(createTrainingDatasetFilter(null, AND, "L"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "1"), SINGLE, "L.L"));
    expected.add(createTrainingDatasetFilter(null, OR, "L.R"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "2"), SINGLE, "L.R.L"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "3"), SINGLE, "L.R.R"));
    Assert.assertEquals(expected.size(), actual.size());
    Assert.assertTrue(expected.containsAll(actual));
    Assert.assertTrue(actual.containsAll(expected));
}

Example 18 with TrainingDataset

use of io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset in project hopsworks by logicalclocks.

the class TestStatisticColumnController method testVerifyStatisticColumnsExistTD.

@Test
public void testVerifyStatisticColumnsExistTD() throws Exception {
    TrainingDataset trainingDataset = new TrainingDataset();
    StatisticsConfigDTO statisticsConfig = new StatisticsConfigDTO();
    statisticsConfig.setColumns(Arrays.asList("ft1", "ft4"));
    trainingDataset.setFeatures(Arrays.asList(new TrainingDatasetFeature(trainingDataset, "ft1", null, null, false, null), new TrainingDatasetFeature(trainingDataset, "ft2", null, null, false, null), new TrainingDatasetFeature(trainingDataset, "ft3", null, null, false, null)));
    TrainingDatasetDTO trainingDatasetDTO = new TrainingDatasetDTO();
    trainingDatasetDTO.setName("td1");
    trainingDatasetDTO.setVersion(1);
    trainingDatasetDTO.setStatisticsConfig(statisticsConfig);
    // should throw exception
    thrown.expect(FeaturestoreException.class);
    statisticColumnController.verifyStatisticColumnsExist(trainingDatasetDTO, trainingDataset);
    // should not throw exception
    statisticsConfig.setColumns(Arrays.asList("ft1", "ft2"));
    statisticColumnController.verifyStatisticColumnsExist(trainingDatasetDTO, trainingDataset);
}

Example 19 with TrainingDataset

use of io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset in project hopsworks by logicalclocks.

the class TrainingDatasetControllerTest method testconvertToFilterEntities_leftLogicRightFilter.

@Test
public void testconvertToFilterEntities_leftLogicRightFilter() throws Exception {
    // fg.feature > 1 and fg.feature > 2
    // "fg.feature > 1" stores as filter logic and "fg.feature > 2" stores as filter
    TrainingDataset trainingDataset = new TrainingDataset();
    Feature f1 = new Feature("test_f", "fg0");
    FilterLogic head = new FilterLogic();
    head.setType(AND);
    FilterLogic left = new FilterLogic();
    Filter left_left = new Filter(f1, GREATER_THAN, "1");
    left.setLeftFilter(left_left);
    left.setType(SINGLE);
    head.setLeftLogic(left);
    Filter right = new Filter(f1, GREATER_THAN, "2");
    head.setRightFilter(right);
    List<TrainingDatasetFilter> actual = target.convertToFilterEntities(head, trainingDataset, "L");
    List<TrainingDatasetFilter> expected = new ArrayList<>();
    expected.add(createTrainingDatasetFilter(null, AND, "L"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "2"), SINGLE, "L.R"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "1"), SINGLE, "L.L"));
    Assert.assertEquals(expected.size(), actual.size());
    Assert.assertTrue(expected.containsAll(actual));
    Assert.assertTrue(actual.containsAll(expected));
}

Example 20 with TrainingDataset

use of io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset in project hopsworks by logicalclocks.

the class TrainingDatasetControllerTest method testconvertToFilterEntities_featureComparison.

@Test
public void testconvertToFilterEntities_featureComparison() throws Exception {
    // fg.feature > fg.otherFeature and fg.feature > fg1.otherFeature
    TrainingDataset trainingDataset = new TrainingDataset();
    Feature f1 = new Feature("test_f", "fg0");
    FilterLogic head = new FilterLogic();
    head.setType(AND);
    FilterValue filterValueLeft = new FilterValue(0, "fg0", "test_f1");
    Filter left = new Filter(f1, GREATER_THAN, filterValueLeft);
    head.setLeftFilter(left);
    FilterValue filterValueRight = new FilterValue(1, "fg1", "test_f2");
    Filter right = new Filter(f1, GREATER_THAN, filterValueRight);
    head.setRightFilter(right);
    List<TrainingDatasetFilter> actual = target.convertToFilterEntities(head, trainingDataset, "L");
    List<TrainingDatasetFilter> expected = new ArrayList<>();
    expected.add(createTrainingDatasetFilter(null, AND, "L"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "test_f1", null, 0), SINGLE, "L.L"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "test_f2", null, 1), SINGLE, "L.R"));
    Assert.assertEquals(expected.size(), actual.size());
    Assert.assertTrue(expected.containsAll(actual));
    Assert.assertTrue(actual.containsAll(expected));
}