Examples with ServingPreparedStatementDTO io.hops.hopsworks.common.featurestore.query.ServingPreparedStatementDTO used on opensource projects

Example 1 with ServingPreparedStatementDTO

use of io.hops.hopsworks.common.featurestore.query.ServingPreparedStatementDTO in project hopsworks by logicalclocks.

the class TrainingDatasetService method getPreparedStatements.

@ApiOperation(value = "Get prepared statements used to generate model serving vector from training dataset query", response = ServingPreparedStatementDTO.class)
@GET
@Path("/{trainingdatasetid}/preparedstatements")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getPreparedStatements(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Id of the trainingdatasetid", required = true) @PathParam("trainingdatasetid") Integer trainingDatsetId, @ApiParam(value = "get batch serving vectors", example = "false") @QueryParam("batch") @DefaultValue("false") boolean batch) throws FeaturestoreException {
    verifyIdProvided(trainingDatsetId);
    Users user = jWTHelper.getUserPrincipal(sc);
    ServingPreparedStatementDTO servingPreparedStatementDTO = preparedStatementBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.PREPAREDSTATEMENTS), project, user, featurestore, trainingDatsetId, batch);
    return Response.ok().entity(servingPreparedStatementDTO).build();
}

Example 2 with ServingPreparedStatementDTO

use of io.hops.hopsworks.common.featurestore.query.ServingPreparedStatementDTO in project hopsworks by logicalclocks.

the class PreparedStatementBuilder method getServingStatements.

private List<ServingPreparedStatementDTO> getServingStatements(TrainingDataset trainingDataset, Project project, Users user, boolean batch) throws FeaturestoreException {
    if (!trainingDataset.isQuery()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRAINING_DATASET_NO_QUERY, Level.FINE, "Inference vector is only available for datasets generated by queries");
    }
    List<ServingPreparedStatementDTO> servingPreparedStatementDTOS = new ArrayList<>();
    List<TrainingDatasetJoin> joins = trainingDatasetController.getJoinsSorted(trainingDataset);
    // Check that all the feature groups still exists, if not throw a reasonable error
    if (trainingDataset.getFeatures().stream().anyMatch(j -> j.getFeatureGroup() == null)) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRAINING_DATASET_QUERY_FG_DELETED, Level.FINE);
    }
    // each join is a feature group, iterate over them.
    for (TrainingDatasetJoin join : joins) {
        Featuregroup featuregroup = join.getFeatureGroup();
        if (!featuregroup.getCachedFeaturegroup().isOnlineEnabled()) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ONLINE_NOT_ENABLED, Level.FINE, "Inference vector is only available for training datasets generated by online enabled " + "feature groups");
        }
        Map<String, Feature> featureGroupFeatures = featuregroupController.getFeatures(featuregroup, project, user).stream().collect(Collectors.toMap(FeatureGroupFeatureDTO::getName, f -> new Feature(f.getName(), ALIAS, f.getType(), f.getPrimary(), f.getDefaultValue(), join.getPrefix())));
        // Identify and create primary key features for this feature group. Primary key features may not be the part of
        // query that generated the training dataset.
        List<Feature> primaryKeys = featureGroupFeatures.values().stream().filter(Feature::isPrimary).collect(Collectors.toList());
        if (primaryKeys.size() == 0) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.PRIMARY_KEY_REQUIRED, Level.FINE, "Inference vector is only available for training datasets generated by feature groups with " + "at least 1 primary key");
        }
        // create td features
        List<Feature> selectFeatures = join.getFeatures().stream().filter(tdf -> !tdf.isLabel()).sorted(Comparator.comparing(TrainingDatasetFeature::getIndex)).map(tdf -> featureGroupFeatures.get(tdf.getName())).collect(Collectors.toList());
        // part of the prepared statement thus don't add to this query.
        if (selectFeatures.size() > 0) {
            // construct query for this feature group
            Query query = new Query(featurestoreController.getOfflineFeaturestoreDbName(featuregroup.getFeaturestore().getProject()), onlineFeaturestoreController.getOnlineFeaturestoreDbName(featuregroup.getFeaturestore().getProject()), featuregroup, ALIAS, selectFeatures);
            // construct ServingPreparedStatementDTO and add to the list
            servingPreparedStatementDTOS.add(buildDTO(query, primaryKeys, featuregroup.getId(), join.getIndex(), batch));
        }
    }
    return servingPreparedStatementDTOS;
}

Example 3 with ServingPreparedStatementDTO

use of io.hops.hopsworks.common.featurestore.query.ServingPreparedStatementDTO in project hopsworks by logicalclocks.

the class PreparedStatementBuilder method buildDTO.

private ServingPreparedStatementDTO buildDTO(Query query, List<Feature> primaryKeys, Integer featureGroupId, Integer statementIndex, boolean batch) throws FeaturestoreException {
    // create primary key prepared statement filters for the query
    List<PreparedStatementParameterDTO> stmtParameters = new ArrayList<>();
    // Change the type of PK to PREPARED_STATEMENT_TYPE. This will avoid having the query constructor
    // adding additional quotes around the ? sign
    primaryKeys.forEach(f -> f.setType(PREPARED_STATEMENT_TYPE));
    // record pk position in the prepared statement - start from 1 as that's how
    // prepared statements work.
    int primaryKeyIndex = 1;
    // First condition doesn't have any "AND"
    // we are guaranteed there is at least one primary key, as no primary key situations are filtered above
    Feature pkFeature = primaryKeys.get(0);
    stmtParameters.add(new PreparedStatementParameterDTO(pkFeature.getName(), primaryKeyIndex++));
    FilterLogic filterLogic;
    if (batch) {
        filterLogic = new FilterLogic(new Filter(primaryKeys, SqlCondition.IN, "?"));
        query.setOrderByFeatures(primaryKeys);
    } else {
        filterLogic = new FilterLogic(new Filter(Arrays.asList(pkFeature), SqlCondition.EQUALS, "?"));
    }
    // Concatenate conditions
    for (int i = 1; i < primaryKeys.size(); i++) {
        pkFeature = primaryKeys.get(i);
        if (!batch) {
            filterLogic = filterLogic.and(new Filter(Arrays.asList(pkFeature), SqlCondition.EQUALS, "?"));
        }
        stmtParameters.add(new PreparedStatementParameterDTO(pkFeature.getName(), primaryKeyIndex++));
    }
    query.setFilter(filterLogic);
    // set prepared statement parameters
    return new ServingPreparedStatementDTO(featureGroupId, statementIndex, stmtParameters, constructorController.generateSQL(query, true).toSqlString(new MysqlSqlDialect(SqlDialect.EMPTY_CONTEXT)).getSql());
}

Example 4 with ServingPreparedStatementDTO

use of io.hops.hopsworks.common.featurestore.query.ServingPreparedStatementDTO in project hopsworks by logicalclocks.

the class PreparedStatementBuilder method build.

public ServingPreparedStatementDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user, Featurestore featurestore, Integer trainingDatasetId, boolean batch) throws FeaturestoreException {
    TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId);
    List<ServingPreparedStatementDTO> servingPreparedStatementDTOs = getServingStatements(trainingDataset, project, user, batch);
    ServingPreparedStatementDTO servingPreparedStatementDTO = new ServingPreparedStatementDTO();
    servingPreparedStatementDTO.setHref(uri(uriInfo, project, featurestore, trainingDataset));
    servingPreparedStatementDTO.setExpand(expand(resourceRequest));
    if (servingPreparedStatementDTO.isExpand()) {
        servingPreparedStatementDTO.setItems(servingPreparedStatementDTOs);
        servingPreparedStatementDTO.setCount((long) servingPreparedStatementDTOs.size());
    }
    return servingPreparedStatementDTO;
}