Examples with Join io.hops.hopsworks.common.featurestore.query.join.Join used on opensource projects

Example 26 with Join

use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.

the class ConstructorController method getHudiAliases.

public List<HudiFeatureGroupAliasDTO> getHudiAliases(Query query, List<HudiFeatureGroupAliasDTO> aliases, Project project, Users user) throws FeaturestoreException, ServiceException {
    if (query.getFeaturegroup().getFeaturegroupType() == FeaturegroupType.CACHED_FEATURE_GROUP && query.getFeaturegroup().getCachedFeaturegroup().getTimeTravelFormat() == TimeTravelFormat.HUDI) {
        CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(query.getFeaturegroup());
        Featuregroup featuregroup = query.getFeaturegroup();
        List<FeatureGroupFeatureDTO> featureGroupFeatureDTOS = cachedFeaturegroupController.getFeaturesDTO(featuregroup, project, user);
        featuregroupDTO.setFeatures(featureGroupFeatureDTOS);
        featuregroupDTO.setLocation(featurestoreUtils.resolveLocationURI(featuregroup.getCachedFeaturegroup().getHiveTbls().getSdId().getLocation()));
        if (query.getLeftFeatureGroupStartTimestamp() == null) {
            aliases.add(new HudiFeatureGroupAliasDTO(query.getAs(), featuregroupDTO, query.getLeftFeatureGroupEndTimestamp()));
        } else {
            aliases.add(new HudiFeatureGroupAliasDTO(query.getAs(), featuregroupDTO, query.getLeftFeatureGroupStartTimestamp(), query.getLeftFeatureGroupEndTimestamp()));
        }
    }
    if (query.getJoins() != null && !query.getJoins().isEmpty()) {
        for (Join join : query.getJoins()) {
            getHudiAliases(join.getRightQuery(), aliases, project, user);
        }
    }
    return aliases;
}

Example 27 with Join

use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.

the class ConstructorController method getOnDemandAliases.

// TODO(Fabio): does it make sense to this in the same pass as where we generate the table nodes?
// or does the code becomes even more complicated?
public List<OnDemandFeatureGroupAliasDTO> getOnDemandAliases(Users user, Project project, Query query, List<OnDemandFeatureGroupAliasDTO> aliases) throws FeaturestoreException, ServiceException {
    if (query.getFeaturegroup().getFeaturegroupType() == FeaturegroupType.ON_DEMAND_FEATURE_GROUP) {
        FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = storageConnectorController.convertToConnectorDTO(user, project, query.getFeaturegroup().getOnDemandFeaturegroup().getFeaturestoreConnector());
        OnDemandFeaturegroupDTO onDemandFeaturegroupDTO = new OnDemandFeaturegroupDTO(query.getFeaturegroup(), featurestoreStorageConnectorDTO);
        try {
            String path = featuregroupController.getFeatureGroupLocation(query.getFeaturegroup());
            onDemandFeaturegroupDTO.setLocation(featurestoreUtils.prependNameNode(path));
        } catch (ServiceDiscoveryException e) {
            throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_NOT_FOUND, Level.SEVERE);
        }
        aliases.add(new OnDemandFeatureGroupAliasDTO(query.getAs(), onDemandFeaturegroupDTO));
    }
    if (query.getJoins() != null && !query.getJoins().isEmpty()) {
        for (Join join : query.getJoins()) {
            getOnDemandAliases(user, project, join.getRightQuery(), aliases);
        }
    }
    return aliases;
}

Example 28 with Join

use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.

the class TestConstructorController method testMakeOfflineQuery_hiveQueryNested.

@Test
public void testMakeOfflineQuery_hiveQueryNested() throws Exception {
    List<Feature> availableLeft = new ArrayList<>();
    availableLeft.add(new Feature("ft1", "fg0", true));
    List<Feature> availableRight = new ArrayList<>();
    availableRight.add(new Feature("ft1", "fg1", true));
    Query leftQuery = new Query("fs1", "project_fs1", fgHudi, "fg0", availableLeft, availableLeft);
    Query rightQuery = new Query("fs1", "project_fs1", fgHudi, "fg1", availableRight, availableRight);
    leftQuery.setHiveEngine(true);
    rightQuery.setHiveEngine(true);
    Join join = new Join(leftQuery, rightQuery, availableLeft, availableRight, JoinType.INNER, null, singleEqualsJoinOperator);
    leftQuery.setJoins(Arrays.asList(join));
    leftQuery.setOrderByFeatures(availableLeft);
    List<Feature> allFeatures = new ArrayList<>(availableLeft);
    allFeatures.addAll(availableRight);
    Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(allFeatures)).thenReturn(allFeatures);
    Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(availableRight)).thenReturn(availableRight);
    String actual = target.makeOfflineQuery(leftQuery);
    String expected = "SELECT `fg0`.`ft1`, `fg1`.`ft1`\n" + "FROM `fs1`.`fgHudi_1` `fg0`\n" + "INNER JOIN `fs1`.`fgHudi_1` `fg1` ON `fg0`.`ft1` = `fg1`.`ft1`\n" + "ORDER BY `fg0`.`ft1`";
    Assert.assertEquals(expected, actual);
}

Example 29 with Join

use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.

the class TestConstructorController method testSingleJoinSQLQueryOnline.

@Test
public void testSingleJoinSQLQueryOnline() throws Exception {
    List<Feature> availableLeft = new ArrayList<>();
    availableLeft.add(new Feature("ft1", "fg1", "Float", null, null));
    List<Feature> availableRight = new ArrayList<>();
    availableRight.add(new Feature("ft1", "fg2", "Float", null, null));
    Query leftQuery = new Query("fs1", "project_fs2", fg1, "fg1", availableLeft, availableLeft);
    Query rightQuery = new Query("fs1", "project_fs1", fg2, "fg2", availableRight, availableRight);
    Join join = new Join(leftQuery, rightQuery, availableLeft, availableLeft, JoinType.INNER, null, singleEqualsJoinOperator);
    leftQuery.setJoins(Arrays.asList(join));
    String query = target.generateSQL(leftQuery, true).toSqlString(new SparkSqlDialect(SqlDialect.EMPTY_CONTEXT)).getSql().replace("\n", " ");
    Assert.assertEquals("SELECT `fg1`.`ft1`, `fg2`.`ft1` FROM `project_fs2`.`fg1_1` `fg1` INNER JOIN " + "`project_fs1`.`fg2_1` `fg2` ON `fg1`.`ft1` = `fg2`.`ft1`", query);
}

Example 30 with Join

use of io.hops.hopsworks.common.featurestore.query.join.Join in project hopsworks by logicalclocks.

the class TestConstructorController method testPrefixFeatureJoins.

@Test
public void testPrefixFeatureJoins() throws Exception {
    List<Feature> availableFirst = new ArrayList<>();
    availableFirst.add(new Feature("ft1", "fg0", "Float", null, null));
    List<Feature> availableSecond = new ArrayList<>();
    availableSecond.add(new Feature("ft2", "fg1", "Float", null, "prefix2_"));
    List<Feature> availableThird = new ArrayList<>();
    availableThird.add(new Feature("fg4_ft4_1", "fg2", "Float", null, "prefix4_"));
    availableThird.add(new Feature("fg4_ft4_2", "fg2", "Float", null, "prefix4_"));
    Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(Mockito.any())).thenReturn(availableSecond, availableThird, Stream.of(availableFirst, availableSecond, availableThird).flatMap(Collection::stream).collect(Collectors.toList()));
    fg1.getCachedFeaturegroup().setTimeTravelFormat(TimeTravelFormat.HUDI);
    fg2.getCachedFeaturegroup().setTimeTravelFormat(TimeTravelFormat.HUDI);
    fg4.getCachedFeaturegroup().setTimeTravelFormat(TimeTravelFormat.HUDI);
    Query leftQuery = new Query("fs1", "project_fs1", fg1, "fg0", availableFirst, availableFirst);
    Query secondQuery = new Query("fs1", "project_fs1", fg2, "fg1", availableSecond, availableSecond);
    Query thirdQuery = new Query("fs1", "project_fs1", fg4, "fg2", fg4Features, fg4Features);
    Join join = new Join(leftQuery, secondQuery, availableFirst, availableSecond, JoinType.INNER, "prefix2_", singleEqualsJoinOperator);
    Join secondJoin = new Join(leftQuery, thirdQuery, availableFirst, availableFirst, JoinType.INNER, "prefix4_", singleEqualsJoinOperator);
    leftQuery.setJoins(Arrays.asList(join, secondJoin));
    String query = target.generateSQL(leftQuery, true).toSqlString(new SparkSqlDialect(SqlDialect.EMPTY_CONTEXT)).getSql().replace("\n", " ");
    Assert.assertEquals("SELECT `fg0`.`ft1`, `fg1`.`ft2` `prefix2_ft2`, " + "`fg2`.`fg4_ft4_1` `prefix4_fg4_ft4_1`, `fg2`.`fg4_ft4_2` `prefix4_fg4_ft4_2` " + "FROM `project_fs1`.`fg1_1` `fg0` INNER JOIN `project_fs1`.`fg2_1` `fg1` ON `fg0`.`ft1` = `fg1`.`ft2` " + "INNER JOIN `project_fs1`.`fg4_1` `fg2` ON `fg0`.`ft1` = `fg2`.`ft1`", query);
}