Examples with Serving io.hops.hopsworks.persistence.entity.serving.Serving used on opensource projects

Example 6 with Serving

use of io.hops.hopsworks.persistence.entity.serving.Serving in project hopsworks by logicalclocks.

the class LocalhostServingController method delete.

/**
 * Deletes a specific serving from a project
 *
 * @param project the project that contains the serving
 * @param id the id of the serving
 * @throws ServingException if the lock could not be acquired
 */
@Override
public void delete(Project project, Integer id) throws ServingException {
    Serving serving = servingFacade.acquireLock(project, id);
    ServingStatusEnum status = getServingStatus(serving);
    // If we reached this point, we just acquired a lock
    if (!status.equals(ServingStatusEnum.STARTING)) {
        killServingInstance(project, serving, false);
    }
    servingFacade.delete(serving);
}

Example 7 with Serving

use of io.hops.hopsworks.persistence.entity.serving.Serving in project hopsworks by logicalclocks.

the class InferenceController method infer.

/**
 * Makes an inference request to a running serving instance
 *
 * @param project the project where the serving is running
 * @param modelName the name of the serving
 * @param modelVersion the version of the serving
 * @param verb the predictiont type (predict, regress, or classify)
 * @param inferenceRequestJson the user-provided JSON payload for the inference request
 * @return a string representation of the inference result
 * @throws InferenceException
 */
public String infer(Project project, String username, String modelName, Integer modelVersion, InferenceVerb verb, String inferenceRequestJson, String authHeader) throws InferenceException, ApiKeyException {
    Serving serving = servingFacade.findByProjectAndName(project, modelName);
    if (serving == null) {
        throw new InferenceException(RESTCodes.InferenceErrorCode.SERVING_NOT_FOUND, Level.FINE, "name: " + modelName);
    }
    if (verb == null) {
        throw new InferenceException(RESTCodes.InferenceErrorCode.MISSING_VERB, Level.FINE);
    }
    if (modelVersion != null && modelVersion < 0) {
        throw new InferenceException(RESTCodes.InferenceErrorCode.BAD_REQUEST, Level.FINE, "Model version must be " + "positive");
    }
    // ServingInferenceController is either localhost or kubernetes inference controller
    Pair<Integer, String> inferenceResult = servingInferenceController.infer(username, serving, modelVersion, verb, inferenceRequestJson, authHeader);
    // Log the inference
    for (InferenceLogger inferenceLogger : inferenceLoggers) {
        try {
            inferenceLogger.logInferenceRequest(serving, inferenceRequestJson, inferenceResult.getL(), inferenceResult.getR());
        } catch (Exception e) {
            // We don't want to fill the logs with inference logging errors
            logger.log(Level.FINE, "Error logging inference for logger: " + inferenceLogger.getClassName(), e);
        }
    }
    // If the inference server returned something different than 200 then throw an exception to the user
    if (inferenceResult.getL() >= 500) {
        logger.log(Level.FINE, "Request error: " + inferenceResult.getL() + " - " + inferenceResult.getR());
        throw new InferenceException(RESTCodes.InferenceErrorCode.SERVING_INSTANCE_INTERNAL, Level.FINE, inferenceResult.getR());
    } else if (inferenceResult.getL() >= 400) {
        logger.log(Level.FINE, "Request error: " + inferenceResult.getL() + " - " + inferenceResult.getR());
        throw new InferenceException(RESTCodes.InferenceErrorCode.SERVING_INSTANCE_BAD_REQUEST, Level.FINE, inferenceResult.getR());
    }
    return inferenceResult.getR();
}

Example 8 with Serving

use of io.hops.hopsworks.persistence.entity.serving.Serving in project hopsworks by logicalclocks.

the class LocalhostServingMonitor method monitor.

@Timeout
public void monitor(Timer timer) {
    try {
        // Get the list of running Localhost Serving instances
        List<Serving> servingList = servingFacade.getLocalhostRunning();
        for (Serving serving : servingList) {
            try {
                Serving dbServing = servingFacade.acquireLock(serving.getProject(), serving.getId());
                ProcessDescriptor.Builder builder = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo");
                if (serving.getModelServer() == ModelServer.TENSORFLOW_SERVING) {
                    builder.addCommand(tfScript);
                }
                if (serving.getModelServer() == ModelServer.PYTHON) {
                    builder.addCommand(sklearnScript);
                }
                ProcessDescriptor processDescriptor = builder.addCommand("alive").addCommand(dbServing.getProject().getName().toLowerCase()).addCommand(dbServing.getName()).ignoreOutErrStreams(true).build();
                LOGGER.log(Level.FINE, processDescriptor.toString());
                try {
                    ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
                    if (processResult.getExitCode() != 0) {
                        // The processes is dead, run the kill script to delete the directory
                        // and update the value in the db
                        Path secretDir = Paths.get(settings.getStagingDir(), SERVING_DIRS + serving.getLocalDir());
                        builder = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo");
                        if (serving.getModelServer() == ModelServer.TENSORFLOW_SERVING) {
                            builder.addCommand(tfScript);
                        }
                        if (serving.getModelServer() == ModelServer.PYTHON) {
                            builder.addCommand(sklearnScript);
                        }
                        processDescriptor = builder.addCommand("kill").addCommand(dbServing.getCid()).addCommand(dbServing.getName()).addCommand(dbServing.getProject().getName().toLowerCase()).addCommand(secretDir.toString()).ignoreOutErrStreams(true).build();
                        LOGGER.log(Level.FINE, processDescriptor.toString());
                        osProcessExecutor.execute(processDescriptor);
                        // If the process succeeded to delete the localDir update the db
                        dbServing.setCid(CID_STOPPED);
                        dbServing.setLocalPort(-1);
                        servingFacade.updateDbObject(dbServing, dbServing.getProject());
                    }
                } catch (IOException e) {
                    LOGGER.log(Level.SEVERE, "Could not clean up serving instance with id: " + serving.getId(), e);
                }
                servingFacade.releaseLock(serving.getProject(), serving.getId());
            } catch (ServingException e) {
                LOGGER.log(Level.INFO, "Error processing serving instance with id: " + serving.getId(), e);
            }
        }
    } catch (Exception e) {
        LOGGER.log(Level.SEVERE, "Got an exception while monitoring servings", e);
    }
}

Example 9 with Serving

use of io.hops.hopsworks.persistence.entity.serving.Serving in project hopsworks by logicalclocks.

the class ServingView method getServingWrapper.

@JsonIgnore
public ServingWrapper getServingWrapper() {
    ServingWrapper servingWrapper = new ServingWrapper(new Serving(id, name, description, modelPath, modelName, modelVersion, predictor, requestedInstances, batchingEnabled, modelServer, servingTool));
    servingWrapper.setKafkaTopicDTO(kafkaTopicDTO);
    return servingWrapper;
}

Example 10 with Serving

use of io.hops.hopsworks.persistence.entity.serving.Serving in project hopsworks by logicalclocks.

the class ServingFacade method releaseLock.

@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
public Serving releaseLock(Project project, Integer id) throws ServingException {
    int retries = 5;
    // Acquire DB read lock on the row
    while (retries > 0) {
        try {
            Serving serving = em.createNamedQuery("Serving.findByProjectAndId", Serving.class).setParameter("project", project).setParameter("id", id).setLockMode(LockModeType.PESSIMISTIC_WRITE).getSingleResult();
            serving.setLockIP(null);
            serving.setLockTimestamp(null);
            return em.merge(serving);
        } catch (LockTimeoutException e) {
            retries--;
        }
    }
    // Lock will be claimed
    throw new ServingException(RESTCodes.ServingErrorCode.LIFECYCLEERRORINT, Level.FINE);
}