use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.
the class TrainingDatasetController method collectFeatures.
// Here we need to pass the list of training dataset joins so that we can rebuild the aliases.
// and handle correctly the case in which a feature group is joined with itself.
public List<TrainingDatasetFeature> collectFeatures(Query query, List<TrainingDatasetFeatureDTO> featureDTOs, TrainingDataset trainingDataset, FeatureView featureView, int featureIndex, List<TrainingDatasetJoin> tdJoins, int joinIndex) throws FeaturestoreException {
List<TrainingDatasetFeature> features = new ArrayList<>();
boolean isLabel = false;
TransformationFunction transformationFunction = null;
for (Feature f : query.getFeatures()) {
if (featureDTOs != null && !featureDTOs.isEmpty()) {
// identify if feature is label
isLabel = featureDTOs.stream().anyMatch(dto -> f.getName().equals(dto.getName()) && dto.getLabel());
// get transformation function for this feature
transformationFunction = getTransformationFunction(f, featureDTOs);
}
features.add(trainingDataset != null ? new TrainingDatasetFeature(trainingDataset, tdJoins.get(joinIndex), query.getFeaturegroup(), f.getName(), f.getType(), featureIndex++, isLabel, transformationFunction) : new TrainingDatasetFeature(featureView, tdJoins.get(joinIndex), query.getFeaturegroup(), f.getName(), f.getType(), featureIndex++, isLabel, transformationFunction));
}
if (query.getJoins() != null) {
for (Join join : query.getJoins()) {
joinIndex++;
List<TrainingDatasetFeature> joinFeatures = collectFeatures(join.getRightQuery(), featureDTOs, trainingDataset, featureView, featureIndex, tdJoins, joinIndex);
features.addAll(joinFeatures);
featureIndex += joinFeatures.size();
}
}
return features;
}
use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.
the class TrainingDatasetController method getTransformationFunction.
private TransformationFunction getTransformationFunction(Feature feature, List<TrainingDatasetFeatureDTO> featureDTOs) throws FeaturestoreException {
TrainingDatasetFeatureDTO featureDTO = featureDTOs.stream().filter(dto -> feature.getName().equals(dto.getFeatureGroupFeatureName())).findFirst().orElse(null);
TransformationFunction transformationFunction = null;
if (featureDTO != null && featureDTO.getTransformationFunction() != null) {
transformationFunction = transformationFunctionFacade.findById(featureDTO.getTransformationFunction().getId()).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRANSFORMATION_FUNCTION_DOES_NOT_EXIST, Level.FINE, "Could not find transformation function with ID" + featureDTO.getTransformationFunction().getId()));
}
return transformationFunction;
}
use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.
the class TransformationFunctionController method delete.
public void delete(Project project, Featurestore featurestore, Users user, Integer transformationFunctionId) throws FeaturestoreException {
TransformationFunction transformationFunction = transformationFunctionFacade.findById(transformationFunctionId).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRANSFORMATION_FUNCTION_DOES_NOT_EXIST, Level.FINE, "Could not find transformation function with ID" + transformationFunctionId));
// Check if trying to delete built in transformation function
if (FeaturestoreConstants.BUILT_IN_TRANSFORMATION_FUNCTION_NAMES.contains(transformationFunction.getName()) && transformationFunction.getVersion() == 1) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ERROR_DELETING_TRANSFORMERFUNCTION, Level.FINE, "Deleting built-in transformation function `" + transformationFunction.getName() + "` with version 1 is not " + "allowed. Create a new version instead.");
}
DistributedFileSystemOps udfso = null;
try {
udfso = dfs.getDfsOps(hdfsUsersController.getHdfsUserName(project, user));
String dirName = getFeatureStoreEntityName(transformationFunction.getName(), transformationFunction.getVersion());
// Construct the directory path
Path dirPath = new Path(getOrCreatePath(featurestore, udfso), dirName);
// delete json files
udfso.rm(dirPath, true);
} catch (IOException e) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ERROR_DELETING_TRANSFORMERFUNCTION, Level.WARNING, "", e.getMessage(), e);
} finally {
dfs.closeDfsClient(udfso);
}
}
use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.
the class TransformationFunctionFacade method findByFeaturestore.
/**
* Retrieves transformation functions by featurestore
*
* @param featurestore
* @param offset
* @return
*/
public AbstractFacade.CollectionInfo findByFeaturestore(Integer offset, Integer limit, Set<? extends AbstractFacade.FilterBy> filters, Set<? extends AbstractFacade.SortBy> sort, Featurestore featurestore) {
String queryStr = buildQuery("SELECT tfn FROM TransformationFunction tfn ", filters, sort, "tfn.featurestore = :featurestore ");
String queryCountStr = buildQuery("SELECT COUNT(tfn.id) FROM TransformationFunction tfn ", filters, sort, "tfn.featurestore = :featurestore ");
Query query = em.createQuery(queryStr, TransformationFunction.class).setParameter("featurestore", featurestore);
Query queryCount = em.createQuery(queryCountStr, TransformationFunction.class).setParameter("featurestore", featurestore);
setFilter(filters, query);
setFilter(filters, queryCount);
setOffsetAndLim(offset, limit, query);
return findAll(offset, limit, filters, query, queryCount);
}
use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.
the class TransformationFunctionResource method register.
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Register transformation function in to a featurestore", response = TransformationFunctionDTO.class)
public Response register(@Context UriInfo uriInfo, @Context SecurityContext sc, TransformationFunctionDTO transformationFunctionDTO) throws IOException, FeaturestoreException {
Users user = jWTHelper.getUserPrincipal(sc);
TransformationFunction transformationFunction = transformationFunctionController.register(user, project, featurestore, transformationFunctionDTO);
TransformationFunctionDTO newTransformationFunctionDTO = transformationFunctionBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.TRANSFORMATIONFUNCTIONS), user, project, featurestore, transformationFunction);
return Response.ok().entity(newTransformationFunctionDTO).build();
}
Aggregations