use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.
the class TestConstructorController method testThreeWayJoinSQLNodeWithFilters.
@Test
public void testThreeWayJoinSQLNodeWithFilters() throws Exception {
List<Feature> availableFirst = new ArrayList<>();
availableFirst.add(new Feature("ft1", "fg0", "Float", null, null));
List<Feature> availableSecond = new ArrayList<>();
availableSecond.add(new Feature("ft2", "fg1", "Float", null, null));
List<Feature> availableThird = new ArrayList<>();
availableThird.add(new Feature("ft1", "fg2", "Float", null, null));
availableThird.add(new Feature("ft2", "fg2", "Float", null, null));
Query leftQuery = new Query("fs1", "project_fs1", fg1, "fg0", availableFirst, availableFirst);
Query secondQuery = new Query("fs1", "project_fs1", fg2, "fg1", availableSecond, availableSecond);
Query thirdQuery = new Query("fs1", "project_fs1", fg3, "fg2", availableThird, availableThird);
FilterLogic firstFilter = new FilterLogic(SqlFilterLogic.AND);
firstFilter.setLeftFilter(new Filter(Arrays.asList(availableFirst.get(0)), SqlCondition.EQUALS, "10"));
FilterLogic rightLogic = new FilterLogic(SqlFilterLogic.OR);
rightLogic.setLeftFilter(new Filter(Arrays.asList(availableThird.get(0)), SqlCondition.EQUALS, "10"));
rightLogic.setRightFilter(new Filter(Arrays.asList(availableThird.get(1)), SqlCondition.EQUALS, "10"));
firstFilter.setRightLogic(rightLogic);
leftQuery.setFilter(firstFilter);
FilterLogic secondFilter = new FilterLogic(SqlFilterLogic.SINGLE);
secondFilter.setLeftFilter(new Filter(Arrays.asList(availableSecond.get(0)), SqlCondition.NOT_EQUALS, "10"));
secondQuery.setFilter(secondFilter);
Join join = new Join(leftQuery, secondQuery, availableFirst, availableSecond, JoinType.INNER, null, singleEqualsJoinOperator);
Join secondJoin = new Join(leftQuery, thirdQuery, availableFirst, availableFirst, JoinType.INNER, null, singleEqualsJoinOperator);
leftQuery.setJoins(Arrays.asList(join, secondJoin));
String query = target.generateSQL(leftQuery, false).toSqlString(new SparkSqlDialect(SqlDialect.EMPTY_CONTEXT)).getSql().replace("\n", " ");
Assert.assertEquals("SELECT `fg0`.`ft1`, `fg1`.`ft2`, `fg2`.`ft1`, `fg2`.`ft2` " + "FROM `fs1`.`fg1_1` `fg0` " + "INNER JOIN `fs1`.`fg2_1` `fg1` ON `fg0`.`ft1` = `fg1`.`ft2` " + "INNER JOIN `fs1`.`fg3_1` `fg2` ON `fg0`.`ft1` = `fg2`.`ft1` " + "WHERE `fg0`.`ft1` = 10 AND (`fg2`.`ft1` = 10 OR `fg2`.`ft2` = 10) AND `fg1`.`ft2` <> 10", query);
}
use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.
the class TestConstructorController method testTreeWayHudiJoinSQLNodeHiveEngine.
@Test
public void testTreeWayHudiJoinSQLNodeHiveEngine() throws Exception {
List<Feature> availableFirst = new ArrayList<>();
availableFirst.add(new Feature("ft1", "fg0", "Float", null, null));
List<Feature> availableSecond = new ArrayList<>();
availableSecond.add(new Feature("ft2", "fg1", "Float", null, null));
List<Feature> availableThird = new ArrayList<>();
availableThird.add(new Feature("ft1", "fg2", "Float", null, null));
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(Mockito.any())).thenReturn(availableSecond, availableThird, Stream.of(availableFirst, availableSecond, availableThird).flatMap(Collection::stream).collect(Collectors.toList()));
fg1.getCachedFeaturegroup().setTimeTravelFormat(TimeTravelFormat.HUDI);
fg2.getCachedFeaturegroup().setTimeTravelFormat(TimeTravelFormat.HUDI);
fg3.getCachedFeaturegroup().setTimeTravelFormat(TimeTravelFormat.HUDI);
Query leftQuery = new Query("fs1", "project_fs1", fg1, "fg0", availableFirst, availableFirst, true);
Query secondQuery = new Query("fs1", "project_fs1", fg2, "fg1", availableSecond, availableSecond, true);
Query thirdQuery = new Query("fs1", "project_fs1", fg3, "fg2", availableThird, availableThird, true);
Join join = new Join(leftQuery, secondQuery, availableFirst, availableSecond, JoinType.INNER, null, singleEqualsJoinOperator);
Join secondJoin = new Join(leftQuery, thirdQuery, availableFirst, availableFirst, JoinType.INNER, null, singleEqualsJoinOperator);
leftQuery.setJoins(Arrays.asList(join, secondJoin));
String query = target.generateSQL(leftQuery, false).toSqlString(new SparkSqlDialect(SqlDialect.EMPTY_CONTEXT)).getSql().replace("\n", " ");
Assert.assertEquals("SELECT `fg0`.`ft1`, `fg1`.`ft2`, `fg2`.`ft1` " + "FROM `fs1`.`fg1_1` `fg0` " + "INNER JOIN `fs1`.`fg2_1` `fg1` ON `fg0`.`ft1` = `fg1`.`ft2` " + "INNER JOIN `fs1`.`fg3_1` `fg2` ON `fg0`.`ft1` = `fg2`.`ft1`", query);
}
use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.
the class TestConstructorController method testSingleJoinOrderBySQLQuery.
@Test
public void testSingleJoinOrderBySQLQuery() throws Exception {
List<Feature> availableLeft = new ArrayList<>();
availableLeft.add(new Feature("ft1", "fg0", "Float", null, null));
List<Feature> availableRight = new ArrayList<>();
availableRight.add(new Feature("ft1", "fg1", "Float", null, null));
Query rightQuery = new Query("fs1", "project_fs1", fg2, "fg0", availableRight, availableRight);
Query leftQuery = new Query("fs1", "project_fs1", fg1, "fg1", availableLeft, availableLeft);
Join join = new Join(leftQuery, rightQuery, availableLeft, availableLeft, JoinType.INNER, null, singleEqualsJoinOperator);
leftQuery.setJoins(Arrays.asList(join));
leftQuery.setOrderByFeatures(availableRight);
String query = target.generateSQL(leftQuery, false).toSqlString(new SparkSqlDialect(SqlDialect.EMPTY_CONTEXT)).getSql().replace("\n", " ");
Assert.assertEquals("SELECT `fg0`.`ft1`, `fg1`.`ft1` FROM `fs1`.`fg1_1` `fg1` INNER JOIN " + "`fs1`.`fg2_1` `fg0` ON `fg1`.`ft1` = `fg0`.`ft1` ORDER BY `fg1`.`ft1`", query);
}
use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.
the class TestConstructorController method testSingleJoinSQLQuery.
@Test
public void testSingleJoinSQLQuery() throws Exception {
List<Feature> availableLeft = new ArrayList<>();
availableLeft.add(new Feature("ft1", "fg0", "Float", null, null));
List<Feature> availableRight = new ArrayList<>();
availableRight.add(new Feature("ft1", "fg1", "Float", null, null));
Query rightQuery = new Query("fs1", "project_fs1", fg2, "fg0", availableRight, availableRight);
Query leftQuery = new Query("fs1", "project_fs1", fg1, "fg1", availableLeft, availableLeft);
Join join = new Join(leftQuery, rightQuery, availableLeft, availableLeft, JoinType.INNER, null, singleEqualsJoinOperator);
leftQuery.setJoins(Arrays.asList(join));
String query = target.generateSQL(leftQuery, false).toSqlString(new SparkSqlDialect(SqlDialect.EMPTY_CONTEXT)).getSql().replace("\n", " ");
Assert.assertEquals("SELECT `fg0`.`ft1`, `fg1`.`ft1` FROM `fs1`.`fg1_1` `fg1` INNER JOIN " + "`fs1`.`fg2_1` `fg0` ON `fg1`.`ft1` = `fg0`.`ft1`", query);
}
use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.
the class TestConstructorController method testNoJoiningKeySingle.
@Test
public void testNoJoiningKeySingle() throws Exception {
ConstructorController constructorController = new ConstructorController();
List<Feature> availableLeft = new ArrayList<>();
availableLeft.add(new Feature("ft1", true));
List<Feature> availableRight = new ArrayList<>();
availableRight.add(new Feature("ft1", true));
Query leftQuery = new Query("fs1", "project_fs1", fg1, "fg1", availableLeft, availableLeft);
Query rightQuery = new Query("fs1", "project_fs1", fg2, "fg1", availableRight, availableRight);
List<SqlCondition> expectedConditionList = Arrays.asList(SqlCondition.EQUALS);
Join join = queryController.extractPrimaryKeysJoin(leftQuery, rightQuery, JoinType.INNER, null);
Assert.assertEquals(1, join.getLeftOn().size());
Assert.assertEquals(1, join.getRightOn().size());
Assert.assertEquals(expectedConditionList, join.getJoinOperator());
}
Aggregations