Examples with TransformationFunction io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction used on opensource projects

Example 6 with TransformationFunction

use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.

the class TestTransformationFunctionController method testDelete_builtin_exception.

@Test
public void testDelete_builtin_exception() {
    List<Optional<TransformationFunction>> fctList = FeaturestoreConstants.BUILT_IN_TRANSFORMATION_FUNCTION_NAMES.stream().map(name -> Optional.of(new TransformationFunction(name, 1))).collect(Collectors.toList());
    Mockito.when(transformationFunctionFacade.findById(1)).thenReturn(fctList.get(0), fctList.get(1), fctList.get(2), fctList.get(3));
    Assert.assertThrows(FeaturestoreException.class, () -> transformationFunctionController.delete(project, fs, user, 1));
    Assert.assertThrows(FeaturestoreException.class, () -> transformationFunctionController.delete(project, fs, user, 1));
    Assert.assertThrows(FeaturestoreException.class, () -> transformationFunctionController.delete(project, fs, user, 1));
    Assert.assertThrows(FeaturestoreException.class, () -> transformationFunctionController.delete(project, fs, user, 1));
}

Example 7 with TransformationFunction

use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.

the class TransformationFunctionController method create.

private Inode create(Users user, Project project, Featurestore featurestore, TransformationFunctionDTO transformationFunctionDTO) throws IOException, FeaturestoreException {
    // if version not provided, get latest and increment
    if (transformationFunctionDTO.getVersion() == null) {
        // returns ordered list by desc version
        List<TransformationFunction> transformationFnPrevious = transformationFunctionFacade.findByNameAndFeaturestoreOrderedDescVersion(transformationFunctionDTO.getName(), featurestore, 0).getItems();
        if (transformationFnPrevious != null && !transformationFnPrevious.isEmpty()) {
            transformationFunctionDTO.setVersion(transformationFnPrevious.get(0).getVersion() + 1);
        } else {
            transformationFunctionDTO.setVersion(1);
        }
    }
    // Check that transformation function doesn't already exists
    if (transformationFunctionFacade.findByNameVersionAndFeaturestore(transformationFunctionDTO.getName(), transformationFunctionDTO.getVersion(), featurestore).isPresent()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRANSFORMATION_FUNCTION_ALREADY_EXISTS, Level.FINE, "Transformation function: " + transformationFunctionDTO.getName() + ", version: " + transformationFunctionDTO.getVersion());
    }
    DistributedFileSystemOps udfso = null;
    try {
        udfso = dfs.getDfsOps(hdfsUsersController.getHdfsUserName(project, user));
        // Create the directory
        Path dirPath = new Path(getOrCreatePath(featurestore, udfso), getFeatureStoreEntityName(transformationFunctionDTO.getName(), transformationFunctionDTO.getVersion()));
        if (!udfso.isDir(dirPath.toString())) {
            udfso.mkdir(dirPath.toString());
        }
        Path filePath = new Path(dirPath, transformationFunctionDTO.getName() + ".json");
        udfso.create(filePath, transformationFunctionDTO.getSourceCodeContent());
        return inodeController.getInodeAtPath(filePath.toString());
    } finally {
        dfs.closeDfsClient(udfso);
    }
}

Example 8 with TransformationFunction

use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.

the class TransformationFunctionFacade method register.

/**
 * Create and persiste a training dataset builder function
 * @param name
 * @param outputType
 * @param version
 * @param featurestore
 * @param inode
 * @return
 */
public TransformationFunction register(String name, String outputType, Integer version, Featurestore featurestore, Date created, Users user, Inode inode) {
    TransformationFunction transformationFunction = new TransformationFunction();
    transformationFunction.setFeaturestore(featurestore);
    transformationFunction.setInode(inode);
    transformationFunction.setName(name);
    transformationFunction.setOutputType(outputType);
    transformationFunction.setVersion(version);
    transformationFunction.setCreated(created);
    transformationFunction.setCreator(user);
    em.persist(transformationFunction);
    em.flush();
    return transformationFunction;
}

Example 9 with TransformationFunction

use of io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction in project hopsworks by logicalclocks.

the class TransformationFunctionBuilder method build.

public TransformationFunctionDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Users user, Project project, Featurestore featurestore, String name, Integer version) throws FeaturestoreException {
    Long counts;
    List<TransformationFunction> transformationFunctions;
    if (name != null) {
        TransformationFunction transformationFunction;
        if (version != null) {
            transformationFunction = transformationFunctionFacade.findByNameVersionAndFeaturestore(name, version, featurestore).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRANSFORMATION_FUNCTION_DOES_NOT_EXIST, Level.FINE, "Could not find transformation function with name " + name + " and version" + version));
        } else {
            transformationFunction = transformationFunctionFacade.findByNameVersionAndFeaturestore(name, 1, featurestore).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRANSFORMATION_FUNCTION_DOES_NOT_EXIST, Level.FINE, "Could not find transformation function with name " + name + " and version" + 1));
        }
        transformationFunctions = Arrays.asList(transformationFunction);
        counts = 1L;
    } else {
        AbstractFacade.CollectionInfo transformationFunction = transformationFunctionFacade.findByFeaturestore(resourceRequest.getOffset(), resourceRequest.getLimit(), resourceRequest.getFilter(), resourceRequest.getSort(), featurestore);
        transformationFunctions = transformationFunction.getItems();
        counts = transformationFunction.getCount();
    }
    TransformationFunctionDTO transformationFunctionDTO = new TransformationFunctionDTO();
    transformationFunctionDTO.setHref(uri(uriInfo, project, featurestore));
    transformationFunctionDTO.setExpand(expand(resourceRequest));
    if (transformationFunctionDTO.isExpand()) {
        List<TransformationFunctionDTO> list = new ArrayList<>();
        for (TransformationFunction t : transformationFunctions) {
            TransformationFunctionDTO build = build(uriInfo, resourceRequest, user, project, featurestore, t);
            list.add(build);
        }
        transformationFunctionDTO.setItems(list);
        transformationFunctionDTO.setCount(counts);
    }
    return transformationFunctionDTO;
}