Examples with JoinDTO io.hops.hopsworks.common.featurestore.query.join.JoinDTO used on opensource projects

Example 1 with JoinDTO

use of io.hops.hopsworks.common.featurestore.query.join.JoinDTO in project hopsworks by logicalclocks.

the class QueryController method convertJoins.

/**
 * Convert the JoinDTOs into the internal representation of the Join object.
 * The returned list will already contain the correct set of joining keys
 * @param leftQuery
 * @param joinDTOS
 * @return
 */
private List<Join> convertJoins(Query leftQuery, List<JoinDTO> joinDTOS, Map<Integer, String> fgAliasLookup, Map<Integer, Featuregroup> fgLookup, Map<Integer, List<Feature>> availableFeatureLookup, boolean pitEnabled) throws FeaturestoreException {
    List<Join> joins = new ArrayList<>();
    for (JoinDTO joinDTO : joinDTOS) {
        if (joinDTO.getQuery() == null) {
            throw new IllegalArgumentException("Subquery not specified");
        }
        // Recursively convert the QueryDTO. Currently we don't support Joins of Joins
        Query rightQuery = convertQueryDTO(joinDTO.getQuery(), fgAliasLookup, fgLookup, availableFeatureLookup, pitEnabled);
        if (joinDTO.getOn() != null && !joinDTO.getOn().isEmpty()) {
            List<Feature> leftOn = joinDTO.getOn().stream().map(f -> new Feature(f.getName())).collect(Collectors.toList());
            List<Feature> rightOn = joinDTO.getOn().stream().map(f -> new Feature(f.getName())).collect(Collectors.toList());
            joins.add(extractLeftRightOn(leftQuery, rightQuery, leftOn, rightOn, joinDTO.getType(), joinDTO.getPrefix()));
        } else if (joinDTO.getLeftOn() != null && !joinDTO.getLeftOn().isEmpty()) {
            List<Feature> leftOn = joinDTO.getLeftOn().stream().map(f -> new Feature(f.getName())).collect(Collectors.toList());
            List<Feature> rightOn = joinDTO.getRightOn().stream().map(f -> new Feature(f.getName())).collect(Collectors.toList());
            joins.add(extractLeftRightOn(leftQuery, rightQuery, leftOn, rightOn, joinDTO.getType(), joinDTO.getPrefix()));
        } else {
            // Only if right feature group is present, extract the primary keys for the join
            joins.add(extractPrimaryKeysJoin(leftQuery, rightQuery, joinDTO.getType(), joinDTO.getPrefix()));
        }
    }
    return joins;
}

Example 2 with JoinDTO

use of io.hops.hopsworks.common.featurestore.query.join.JoinDTO in project hopsworks by logicalclocks.

the class TestPitJoinController method testIsPitEnabledTrueQueryDTO.

@Test
public void testIsPitEnabledTrueQueryDTO() {
    FeaturegroupDTO fg1 = new FeaturegroupDTO("ts");
    FeaturegroupDTO fg2 = new FeaturegroupDTO("ts");
    JoinDTO join1 = new JoinDTO(new QueryDTO(fg1, null), null, null);
    JoinDTO join2 = new JoinDTO(new QueryDTO(fg2, null), null, null);
    List<JoinDTO> joins = Arrays.asList(join1, join2);
    QueryDTO leftQuery = new QueryDTO(new FeaturegroupDTO("ts"), new ArrayList<>(), joins);
    Assert.assertEquals(true, pitJoinController.isPitEnabled(leftQuery));
}

Example 3 with JoinDTO

use of io.hops.hopsworks.common.featurestore.query.join.JoinDTO in project hopsworks by logicalclocks.

the class TestPitJoinController method testIsPitEnabledFalseQueryDTO.

@Test
public void testIsPitEnabledFalseQueryDTO() {
    FeaturegroupDTO fg1 = new FeaturegroupDTO("ts");
    // not event time enabled fg, therefore should be pitEnabled should be false
    FeaturegroupDTO fg2 = new FeaturegroupDTO();
    JoinDTO join1 = new JoinDTO(new QueryDTO(fg1, null), null, null);
    JoinDTO join2 = new JoinDTO(new QueryDTO(fg2, null), null, null);
    List<JoinDTO> joins = Arrays.asList(join1, join2);
    QueryDTO leftQuery = new QueryDTO(new FeaturegroupDTO("ts"), new ArrayList<>(), joins);
    Assert.assertEquals(false, pitJoinController.isPitEnabled(leftQuery));
}

Example 4 with JoinDTO

use of io.hops.hopsworks.common.featurestore.query.join.JoinDTO in project hopsworks by logicalclocks.

the class QueryController method populateFgLookupTables.

public int populateFgLookupTables(QueryDTO queryDTO, int fgId, Map<Integer, String> fgAliasLookup, Map<Integer, Featuregroup> fgLookup, Map<Integer, List<Feature>> availableFeatureLookup, Project project, Users user, String prefix) throws FeaturestoreException {
    // go into depth first
    if (queryDTO.getJoins() != null && !queryDTO.getJoins().isEmpty()) {
        for (JoinDTO join : queryDTO.getJoins()) {
            fgId = populateFgLookupTables(join.getQuery(), fgId, fgAliasLookup, fgLookup, availableFeatureLookup, project, user, join.getPrefix());
            fgId++;
        }
    }
    Featuregroup fg = validateFeaturegroupDTO(queryDTO.getLeftFeatureGroup());
    fgLookup.put(fg.getId(), fg);
    fgAliasLookup.put(fg.getId(), generateAs(fgId));
    List<Feature> availableFeatures = featuregroupController.getFeatures(fg, project, user).stream().map(f -> new Feature(f.getName(), fgAliasLookup.get(fg.getId()), f.getType(), f.getDefaultValue(), f.getPrimary(), fg, prefix)).collect(Collectors.toList());
    availableFeatureLookup.put(fg.getId(), availableFeatures);
    return fgId;
}

Example 5 with JoinDTO

use of io.hops.hopsworks.common.featurestore.query.join.JoinDTO in project hopsworks by logicalclocks.

the class TestConstructorController method testHandleJoiningKeyRightSide.

@Test
public void testHandleJoiningKeyRightSide() throws Exception {
    // When specifying the "on" condition we remove the duplicates for the joining condition
    // there was a bug for which if the joining feature was selected only on the right side of the join,
    // it was removed. The joining feature should be removed only if it's present on both side,
    // i.e. it's a duplicate. This test make sure we don't regress.
    Mockito.when(featuregroupController.getFeatures(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(fg1FeaturesDTO, fg2FeaturesDTO);
    Mockito.when(featuregroupFacade.findById(Mockito.any())).thenReturn(Optional.of(fg1), Optional.of(fg2));
    Mockito.when(featurestoreFacade.getHiveDbName(Mockito.any())).thenReturn("fg1", "fg2");
    FeaturegroupDTO fg1 = new FeaturegroupDTO();
    fg1.setId(1);
    FeaturegroupDTO fg2 = new FeaturegroupDTO();
    fg2.setId(2);
    List<FeatureGroupFeatureDTO> leftRequestedFeatures = new ArrayList<>();
    leftRequestedFeatures.add(new FeatureGroupFeatureDTO("fg1_ft2"));
    List<FeatureGroupFeatureDTO> rightRequestedFeatures = new ArrayList<>();
    rightRequestedFeatures.addAll(Arrays.asList(new FeatureGroupFeatureDTO("fg2_ft2"), new FeatureGroupFeatureDTO("pr")));
    QueryDTO rightQueryDTO = new QueryDTO(fg2, rightRequestedFeatures);
    JoinDTO joinDTO = new JoinDTO(rightQueryDTO, null, null);
    QueryDTO queryDTO = new QueryDTO(fg1, leftRequestedFeatures, Arrays.asList(joinDTO));
    Map<Integer, String> fgAliasLookup = new HashMap<>();
    Map<Integer, Featuregroup> fgLookup = new HashMap<>();
    Map<Integer, List<Feature>> availableFeatureLookup = new HashMap<>();
    queryController.populateFgLookupTables(queryDTO, 1, fgAliasLookup, fgLookup, availableFeatureLookup, project, user, null);
    Query query = queryController.convertQueryDTO(queryDTO, fgAliasLookup, fgLookup, availableFeatureLookup, false);
    List<Feature> extractedFeatures = target.collectFeatures(query);
    // Make sure both features have been returned.
    // It's going to be 3 as the feature "pr" will be identified as primary key and joining key
    // so it's not going to be duplicated
    Assert.assertEquals(3, extractedFeatures.size());
    // Make sure the method sets the feature group name
    Assert.assertTrue(extractedFeatures.get(0).getFgAlias(false).equals("fg1") || extractedFeatures.get(0).getFgAlias(false).equals("fg2"));
}