Examples with ExperimentDTO io.hops.hopsworks.api.experiments.dto.ExperimentDTO used on opensource projects

Example 1 with ExperimentDTO

use of io.hops.hopsworks.api.experiments.dto.ExperimentDTO in project hopsworks by logicalclocks.

the class ExperimentsBuilder method build.

// Build specific
public ExperimentDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user, Map<Long, ExperimentsEndpointDTO> endpoints, ProvStateDTO fileProvenanceHit) throws ExperimentsException, DatasetException, ProvenanceException, MetadataException, GenericException {
    ExperimentDTO experimentDTO = new ExperimentDTO();
    ExperimentsEndpointDTO endpoint = endpoints.get(fileProvenanceHit.getProjectInodeId());
    if (endpoint == null) {
        // no endpoint - no results
        return null;
    }
    uri(experimentDTO, uriInfo, project, endpoint, fileProvenanceHit);
    if (expand(experimentDTO, resourceRequest).isExpand()) {
        if (fileProvenanceHit.getXattrs() != null && fileProvenanceHit.getXattrs().containsKey(EXPERIMENT_SUMMARY_XATTR_NAME)) {
            JSONObject summary = new JSONObject(fileProvenanceHit.getXattrs().get(EXPERIMENT_SUMMARY_XATTR_NAME));
            ExperimentDTO experimentSummary = experimentConverter.unmarshalDescription(summary.toString());
            experimentDTO.setStarted(fileProvenanceHit.getCreateTime());
            boolean updateNeeded = false;
            // update experiment that failed to set experiment status
            if ((experimentSummary.getState() == null || runningState(experimentSummary)) && Provenance.AppState.valueOf(fileProvenanceHit.getAppState().getCurrentState().name()).isFinalState()) {
                updateNeeded = true;
                experimentSummary.setState(fileProvenanceHit.getAppState().getCurrentState().name());
                experimentSummary.setFinished(fileProvenanceHit.getAppState().getFinishTime());
            }
            experimentDTO.setState(experimentSummary.getState());
            experimentDTO.setFinished(experimentSummary.getFinished());
            if (updateNeeded) {
                experimentsController.attachExperiment(user, project, fileProvenanceHit.getMlId(), experimentSummary);
            }
            if (fileProvenanceHit.getXattrs().containsKey(EXPERIMENT_MODEL_XATTR_NAME)) {
                ModelXAttr model = experimentConverter.unmarshal(fileProvenanceHit.getXattrs().get(EXPERIMENT_MODEL_XATTR_NAME), ModelXAttr.class);
                experimentDTO.setModel(model.getId());
                experimentDTO.setModelProjectName(model.getProjectName());
            }
            experimentDTO.setId(experimentSummary.getId());
            experimentDTO.setName(experimentSummary.getName());
            experimentDTO.setUserFullName(experimentSummary.getUserFullName());
            experimentDTO.setMetric(experimentSummary.getMetric());
            experimentDTO.setDescription(experimentSummary.getDescription());
            experimentDTO.setExperimentType(experimentSummary.getExperimentType());
            experimentDTO.setFunction(experimentSummary.getFunction());
            experimentDTO.setDirection(experimentSummary.getDirection());
            experimentDTO.setOptimizationKey(experimentSummary.getOptimizationKey());
            experimentDTO.setJobName(experimentSummary.getJobName());
            experimentDTO.setAppId(experimentSummary.getAppId());
            experimentDTO.setBestDir(experimentSummary.getBestDir());
            experimentDTO.setEnvironment(experimentSummary.getEnvironment());
            experimentDTO.setProgram(experimentSummary.getProgram());
            experimentDTO.setTensorboard(tensorBoardBuilder.build(uriInfo, resourceRequest.get(ResourceRequest.Name.TENSORBOARD), project, fileProvenanceHit.getMlId()));
            experimentDTO.setResults(experimentResultsBuilder.build(uriInfo, resourceRequest.get(ResourceRequest.Name.RESULTS), project, fileProvenanceHit.getMlId()));
        } else {
            return null;
        }
    }
    return experimentDTO;
}

Example 2 with ExperimentDTO

use of io.hops.hopsworks.api.experiments.dto.ExperimentDTO in project hopsworks by logicalclocks.

the class ExperimentsController method attachExperiment.

public void attachExperiment(Users user, Project project, String id, ExperimentDTO experimentSummary) throws DatasetException, ProvenanceException, MetadataException, ExperimentsException {
    String usersFullName = user.getFname() + " " + user.getLname();
    experimentSummary.setUserFullName(usersFullName);
    String experimentPath = Utils.getProjectPath(project.getName()) + Settings.HOPS_EXPERIMENTS_DATASET + "/" + id;
    if (experimentSummary.getFinished() == null && experimentSummary.getDuration() != null) {
        ProvStateDTO fileState = getExperiment(project, id);
        if (fileState != null) {
            experimentSummary.setFinished(fileState.getCreateTime() + experimentSummary.getDuration());
        }
    }
    if (!Strings.isNullOrEmpty(experimentSummary.getAppId())) {
        byte[] appIdBytes = experimentSummary.getAppId().getBytes(StandardCharsets.UTF_8);
        xattrCtrl.upsertProvXAttr(project, user, experimentPath, ExperimentsBuilder.EXPERIMENT_APP_ID_XATTR_NAME, appIdBytes);
    }
    String hdfsUserName = hdfsUsersController.getHdfsUserName(project, user);
    DistributedFileSystemOps udfso = dfs.getDfsOps(hdfsUserName);
    try {
        byte[] storedExpB = xattrCtrl.getProvXAttr(udfso, experimentPath, ExperimentsBuilder.EXPERIMENT_SUMMARY_XATTR_NAME);
        ExperimentDTO storedExp = null;
        if (storedExpB != null) {
            storedExp = experimentConverter.unmarshal(new String(storedExpB, StandardCharsets.UTF_8), ExperimentDTO.class);
        }
        ExperimentDTO experiment = ExperimentDTO.mergeExperiment(experimentSummary, storedExp);
        byte[] experimentB = experimentConverter.marshal(experiment);
        xattrCtrl.upsertProvXAttr(udfso, experimentPath, ExperimentsBuilder.EXPERIMENT_SUMMARY_XATTR_NAME, experimentB);
    } finally {
        if (udfso != null) {
            dfs.closeDfsClient(udfso);
        }
    }
}

Example 3 with ExperimentDTO

use of io.hops.hopsworks.api.experiments.dto.ExperimentDTO in project hopsworks by logicalclocks.

the class ExperimentsBuilder method build.

// Build collection
public ExperimentDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user) throws ExperimentsException {
    ExperimentDTO dto = new ExperimentDTO();
    uri(dto, uriInfo, project);
    expand(dto, resourceRequest);
    dto.setCount(0l);
    validatePagination(resourceRequest);
    if (dto.isExpand()) {
        try {
            Pair<ProvStateParamBuilder, Map<Long, ExperimentsEndpointDTO>> provFilesParamBuilder = buildExperimentProvenanceParams(project, resourceRequest);
            if (provFilesParamBuilder.getValue1().isEmpty()) {
                // no endpoint - no results
                return dto;
            }
            ProvStateDTO fileState = provenanceController.provFileStateList(project, provFilesParamBuilder.getValue0());
            if (fileState != null) {
                List<ProvStateDTO> experiments = fileState.getItems();
                dto.setCount(fileState.getCount());
                if (experiments != null && !experiments.isEmpty()) {
                    for (ProvStateDTO fileProvStateHit : experiments) {
                        ExperimentDTO experimentDTO = build(uriInfo, resourceRequest, project, user, provFilesParamBuilder.getValue1(), fileProvStateHit);
                        if (experimentDTO != null) {
                            dto.addItem(experimentDTO);
                        }
                    }
                }
            }
        } catch (ExperimentsException | DatasetException | ProvenanceException | MetadataException | GenericException e) {
            if (e instanceof ProvenanceException && ProvHelper.missingMappingForField((ProvenanceException) e)) {
                LOGGER.log(Level.WARNING, "Could not find elastic mapping for experiments query", e);
                return dto;
            } else {
                throw new ExperimentsException(RESTCodes.ExperimentsErrorCode.EXPERIMENT_LIST_FAILED, Level.FINE, "Unable to list experiments for project " + project.getName(), e.getMessage(), e);
            }
        }
    }
    return dto;
}

Example 4 with ExperimentDTO

use of io.hops.hopsworks.api.experiments.dto.ExperimentDTO in project hopsworks by logicalclocks.

the class ExperimentsResource method get.

@ApiOperation(value = "Get an experiment", response = ExperimentDTO.class)
@GET
@Path("{id}")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response get(@PathParam("id") String id, @BeanParam ExpansionBeanParam expansions, @Context UriInfo uriInfo, @Context SecurityContext sc) throws ExperimentsException, DatasetException, ProvenanceException, MetadataException, GenericException {
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXPERIMENTS);
    resourceRequest.setExpansions(expansions.getResources());
    ProvStateDTO fileState = experimentsController.getExperiment(project, id);
    Users user = jwtHelper.getUserPrincipal(sc);
    if (fileState != null) {
        Map<Long, ExperimentsEndpointDTO> endpoints = new HashMap<>();
        endpoints.put(project.getInode().getId(), experimentsController.getExperimentsEndpoint(project));
        ExperimentDTO dto = experimentsBuilder.build(uriInfo, resourceRequest, project, user, endpoints, fileState);
        if (dto == null) {
            throw new GenericException(RESTCodes.GenericErrorCode.NOT_AUTHORIZED_TO_ACCESS, Level.FINE);
        } else {
            return Response.ok().entity(dto).build();
        }
    } else {
        throw new ExperimentsException(RESTCodes.ExperimentsErrorCode.EXPERIMENT_NOT_FOUND, Level.FINE);
    }
}

Example 5 with ExperimentDTO

use of io.hops.hopsworks.api.experiments.dto.ExperimentDTO in project hopsworks by logicalclocks.

the class ExperimentsResource method getAll.

@ApiOperation(value = "Get a list of all experiments for this project", response = ExperimentDTO.class)
@GET
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getAll(@BeanParam Pagination pagination, @BeanParam ExperimentsBeanParam experimentsBeanParam, @Context UriInfo uriInfo, @Context SecurityContext sc) throws ExperimentsException {
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXPERIMENTS);
    resourceRequest.setOffset(pagination.getOffset());
    resourceRequest.setLimit(pagination.getLimit());
    resourceRequest.setFilter(experimentsBeanParam.getFilter());
    resourceRequest.setSort(experimentsBeanParam.getSortBySet());
    resourceRequest.setExpansions(experimentsBeanParam.getExpansions().getResources());
    Users user = jwtHelper.getUserPrincipal(sc);
    ExperimentDTO dto = experimentsBuilder.build(uriInfo, resourceRequest, project, user);
    return Response.ok().entity(dto).build();
}