use of com.google.cloud.aiplatform.v1.MachineSpec in project java-aiplatform by googleapis.
the class CreateHyperparameterTuningJobSample method createHyperparameterTuningJobSample.
static void createHyperparameterTuningJobSample(String project, String displayName, String containerImageUri) throws IOException {
JobServiceSettings settings = JobServiceSettings.newBuilder().setEndpoint("us-central1-aiplatform.googleapis.com:443").build();
String location = "us-central1";
// the "close" method on the client to safely clean up any remaining background resources.
try (JobServiceClient client = JobServiceClient.create(settings)) {
StudySpec.MetricSpec metric0 = StudySpec.MetricSpec.newBuilder().setMetricId("accuracy").setGoal(StudySpec.MetricSpec.GoalType.MAXIMIZE).build();
StudySpec.ParameterSpec.DoubleValueSpec doubleValueSpec = StudySpec.ParameterSpec.DoubleValueSpec.newBuilder().setMinValue(0.001).setMaxValue(0.1).build();
StudySpec.ParameterSpec parameter0 = StudySpec.ParameterSpec.newBuilder().setParameterId("lr").setDoubleValueSpec(doubleValueSpec).build();
StudySpec studySpec = StudySpec.newBuilder().addMetrics(metric0).addParameters(parameter0).build();
MachineSpec machineSpec = MachineSpec.newBuilder().setMachineType("n1-standard-4").setAcceleratorType(AcceleratorType.NVIDIA_TESLA_K80).setAcceleratorCount(1).build();
ContainerSpec containerSpec = ContainerSpec.newBuilder().setImageUri(containerImageUri).build();
WorkerPoolSpec workerPoolSpec0 = WorkerPoolSpec.newBuilder().setMachineSpec(machineSpec).setReplicaCount(1).setContainerSpec(containerSpec).build();
CustomJobSpec trialJobSpec = CustomJobSpec.newBuilder().addWorkerPoolSpecs(workerPoolSpec0).build();
HyperparameterTuningJob hyperparameterTuningJob = HyperparameterTuningJob.newBuilder().setDisplayName(displayName).setMaxTrialCount(2).setParallelTrialCount(1).setMaxFailedTrialCount(1).setStudySpec(studySpec).setTrialJobSpec(trialJobSpec).build();
LocationName parent = LocationName.of(project, location);
HyperparameterTuningJob response = client.createHyperparameterTuningJob(parent, hyperparameterTuningJob);
System.out.format("response: %s\n", response);
System.out.format("Name: %s\n", response.getName());
}
}
use of com.google.cloud.aiplatform.v1.MachineSpec in project java-aiplatform by googleapis.
the class CreateTrainingPipelineCustomTrainingManagedDatasetSample method createTrainingPipelineCustomTrainingManagedDatasetSample.
static void createTrainingPipelineCustomTrainingManagedDatasetSample(String project, String displayName, String modelDisplayName, String datasetId, String annotationSchemaUri, String trainingContainerSpecImageUri, String modelContainerSpecImageUri, String baseOutputUriPrefix) throws IOException {
PipelineServiceSettings settings = PipelineServiceSettings.newBuilder().setEndpoint("us-central1-aiplatform.googleapis.com:443").build();
String location = "us-central1";
// the "close" method on the client to safely clean up any remaining background resources.
try (PipelineServiceClient client = PipelineServiceClient.create(settings)) {
JsonArray jsonArgs = new JsonArray();
jsonArgs.add("--model-dir=$(AIP_MODEL_DIR)");
// training_task_inputs
JsonObject jsonTrainingContainerSpec = new JsonObject();
jsonTrainingContainerSpec.addProperty("imageUri", trainingContainerSpecImageUri);
// AIP_MODEL_DIR is set by the service according to baseOutputDirectory.
jsonTrainingContainerSpec.add("args", jsonArgs);
JsonObject jsonMachineSpec = new JsonObject();
jsonMachineSpec.addProperty("machineType", "n1-standard-8");
JsonObject jsonTrainingWorkerPoolSpec = new JsonObject();
jsonTrainingWorkerPoolSpec.addProperty("replicaCount", 1);
jsonTrainingWorkerPoolSpec.add("machineSpec", jsonMachineSpec);
jsonTrainingWorkerPoolSpec.add("containerSpec", jsonTrainingContainerSpec);
JsonArray jsonWorkerPoolSpecs = new JsonArray();
jsonWorkerPoolSpecs.add(jsonTrainingWorkerPoolSpec);
JsonObject jsonBaseOutputDirectory = new JsonObject();
jsonBaseOutputDirectory.addProperty("outputUriPrefix", baseOutputUriPrefix);
JsonObject jsonTrainingTaskInputs = new JsonObject();
jsonTrainingTaskInputs.add("workerPoolSpecs", jsonWorkerPoolSpecs);
jsonTrainingTaskInputs.add("baseOutputDirectory", jsonBaseOutputDirectory);
Value.Builder trainingTaskInputsBuilder = Value.newBuilder();
JsonFormat.parser().merge(jsonTrainingTaskInputs.toString(), trainingTaskInputsBuilder);
Value trainingTaskInputs = trainingTaskInputsBuilder.build();
// model_to_upload
ModelContainerSpec modelContainerSpec = ModelContainerSpec.newBuilder().setImageUri(modelContainerSpecImageUri).build();
Model model = Model.newBuilder().setDisplayName(modelDisplayName).setContainerSpec(modelContainerSpec).build();
GcsDestination gcsDestination = GcsDestination.newBuilder().setOutputUriPrefix(baseOutputUriPrefix).build();
// input_data_config
InputDataConfig inputDataConfig = InputDataConfig.newBuilder().setDatasetId(datasetId).setAnnotationSchemaUri(annotationSchemaUri).setGcsDestination(gcsDestination).build();
// training_task_definition
String customTaskDefinition = "gs://google-cloud-aiplatform/schema/trainingjob/definition/custom_task_1.0.0.yaml";
TrainingPipeline trainingPipeline = TrainingPipeline.newBuilder().setDisplayName(displayName).setInputDataConfig(inputDataConfig).setTrainingTaskDefinition(customTaskDefinition).setTrainingTaskInputs(trainingTaskInputs).setModelToUpload(model).build();
LocationName parent = LocationName.of(project, location);
TrainingPipeline response = client.createTrainingPipeline(parent, trainingPipeline);
System.out.format("response: %s\n", response);
System.out.format("Name: %s\n", response.getName());
}
}
use of com.google.cloud.aiplatform.v1.MachineSpec in project java-aiplatform by googleapis.
the class CreateBatchPredictionJobVideoClassificationSample method createBatchPredictionJobVideoClassification.
static void createBatchPredictionJobVideoClassification(String batchPredictionDisplayName, String modelId, String gcsSourceUri, String gcsDestinationOutputUriPrefix, String project) throws IOException {
JobServiceSettings jobServiceSettings = JobServiceSettings.newBuilder().setEndpoint("us-central1-aiplatform.googleapis.com:443").build();
// the "close" method on the client to safely clean up any remaining background resources.
try (JobServiceClient jobServiceClient = JobServiceClient.create(jobServiceSettings)) {
String location = "us-central1";
LocationName locationName = LocationName.of(project, location);
VideoClassificationPredictionParams modelParamsObj = VideoClassificationPredictionParams.newBuilder().setConfidenceThreshold(((float) 0.5)).setMaxPredictions(10000).setSegmentClassification(true).setShotClassification(true).setOneSecIntervalClassification(true).build();
Value modelParameters = ValueConverter.toValue(modelParamsObj);
ModelName modelName = ModelName.of(project, location, modelId);
GcsSource.Builder gcsSource = GcsSource.newBuilder();
gcsSource.addUris(gcsSourceUri);
InputConfig inputConfig = InputConfig.newBuilder().setInstancesFormat("jsonl").setGcsSource(gcsSource).build();
GcsDestination gcsDestination = GcsDestination.newBuilder().setOutputUriPrefix(gcsDestinationOutputUriPrefix).build();
OutputConfig outputConfig = OutputConfig.newBuilder().setPredictionsFormat("jsonl").setGcsDestination(gcsDestination).build();
BatchPredictionJob batchPredictionJob = BatchPredictionJob.newBuilder().setDisplayName(batchPredictionDisplayName).setModel(modelName.toString()).setModelParameters(modelParameters).setInputConfig(inputConfig).setOutputConfig(outputConfig).build();
BatchPredictionJob batchPredictionJobResponse = jobServiceClient.createBatchPredictionJob(locationName, batchPredictionJob);
System.out.println("Create Batch Prediction Job Video Classification Response");
System.out.format("\tName: %s\n", batchPredictionJobResponse.getName());
System.out.format("\tDisplay Name: %s\n", batchPredictionJobResponse.getDisplayName());
System.out.format("\tModel %s\n", batchPredictionJobResponse.getModel());
System.out.format("\tModel Parameters: %s\n", batchPredictionJobResponse.getModelParameters());
System.out.format("\tState: %s\n", batchPredictionJobResponse.getState());
System.out.format("\tCreate Time: %s\n", batchPredictionJobResponse.getCreateTime());
System.out.format("\tStart Time: %s\n", batchPredictionJobResponse.getStartTime());
System.out.format("\tEnd Time: %s\n", batchPredictionJobResponse.getEndTime());
System.out.format("\tUpdate Time: %s\n", batchPredictionJobResponse.getUpdateTime());
System.out.format("\tLabels: %s\n", batchPredictionJobResponse.getLabelsMap());
InputConfig inputConfigResponse = batchPredictionJobResponse.getInputConfig();
System.out.println("\tInput Config");
System.out.format("\t\tInstances Format: %s\n", inputConfigResponse.getInstancesFormat());
GcsSource gcsSourceResponse = inputConfigResponse.getGcsSource();
System.out.println("\t\tGcs Source");
System.out.format("\t\t\tUris %s\n", gcsSourceResponse.getUrisList());
BigQuerySource bigQuerySource = inputConfigResponse.getBigquerySource();
System.out.println("\t\tBigquery Source");
System.out.format("\t\t\tInput_uri: %s\n", bigQuerySource.getInputUri());
OutputConfig outputConfigResponse = batchPredictionJobResponse.getOutputConfig();
System.out.println("\tOutput Config");
System.out.format("\t\tPredictions Format: %s\n", outputConfigResponse.getPredictionsFormat());
GcsDestination gcsDestinationResponse = outputConfigResponse.getGcsDestination();
System.out.println("\t\tGcs Destination");
System.out.format("\t\t\tOutput Uri Prefix: %s\n", gcsDestinationResponse.getOutputUriPrefix());
BigQueryDestination bigQueryDestination = outputConfigResponse.getBigqueryDestination();
System.out.println("\t\tBig Query Destination");
System.out.format("\t\t\tOutput Uri: %s\n", bigQueryDestination.getOutputUri());
BatchDedicatedResources batchDedicatedResources = batchPredictionJobResponse.getDedicatedResources();
System.out.println("\tBatch Dedicated Resources");
System.out.format("\t\tStarting Replica Count: %s\n", batchDedicatedResources.getStartingReplicaCount());
System.out.format("\t\tMax Replica Count: %s\n", batchDedicatedResources.getMaxReplicaCount());
MachineSpec machineSpec = batchDedicatedResources.getMachineSpec();
System.out.println("\t\tMachine Spec");
System.out.format("\t\t\tMachine Type: %s\n", machineSpec.getMachineType());
System.out.format("\t\t\tAccelerator Type: %s\n", machineSpec.getAcceleratorType());
System.out.format("\t\t\tAccelerator Count: %s\n", machineSpec.getAcceleratorCount());
ManualBatchTuningParameters manualBatchTuningParameters = batchPredictionJobResponse.getManualBatchTuningParameters();
System.out.println("\tManual Batch Tuning Parameters");
System.out.format("\t\tBatch Size: %s\n", manualBatchTuningParameters.getBatchSize());
OutputInfo outputInfo = batchPredictionJobResponse.getOutputInfo();
System.out.println("\tOutput Info");
System.out.format("\t\tGcs Output Directory: %s\n", outputInfo.getGcsOutputDirectory());
System.out.format("\t\tBigquery Output Dataset: %s\n", outputInfo.getBigqueryOutputDataset());
Status status = batchPredictionJobResponse.getError();
System.out.println("\tError");
System.out.format("\t\tCode: %s\n", status.getCode());
System.out.format("\t\tMessage: %s\n", status.getMessage());
List<Any> details = status.getDetailsList();
for (Status partialFailure : batchPredictionJobResponse.getPartialFailuresList()) {
System.out.println("\tPartial Failure");
System.out.format("\t\tCode: %s\n", partialFailure.getCode());
System.out.format("\t\tMessage: %s\n", partialFailure.getMessage());
List<Any> partialFailureDetailsList = partialFailure.getDetailsList();
}
ResourcesConsumed resourcesConsumed = batchPredictionJobResponse.getResourcesConsumed();
System.out.println("\tResources Consumed");
System.out.format("\t\tReplica Hours: %s\n", resourcesConsumed.getReplicaHours());
CompletionStats completionStats = batchPredictionJobResponse.getCompletionStats();
System.out.println("\tCompletion Stats");
System.out.format("\t\tSuccessful Count: %s\n", completionStats.getSuccessfulCount());
System.out.format("\t\tFailed Count: %s\n", completionStats.getFailedCount());
System.out.format("\t\tIncomplete Count: %s\n", completionStats.getIncompleteCount());
}
}
use of com.google.cloud.aiplatform.v1.MachineSpec in project java-aiplatform by googleapis.
the class CreateBatchPredictionJobVideoObjectTrackingSample method batchPredictionJobVideoObjectTracking.
static void batchPredictionJobVideoObjectTracking(String batchPredictionDisplayName, String modelId, String gcsSourceUri, String gcsDestinationOutputUriPrefix, String project) throws IOException {
JobServiceSettings jobServiceSettings = JobServiceSettings.newBuilder().setEndpoint("us-central1-aiplatform.googleapis.com:443").build();
// the "close" method on the client to safely clean up any remaining background resources.
try (JobServiceClient jobServiceClient = JobServiceClient.create(jobServiceSettings)) {
String location = "us-central1";
LocationName locationName = LocationName.of(project, location);
ModelName modelName = ModelName.of(project, location, modelId);
VideoObjectTrackingPredictionParams modelParamsObj = VideoObjectTrackingPredictionParams.newBuilder().setConfidenceThreshold(((float) 0.5)).build();
Value modelParameters = ValueConverter.toValue(modelParamsObj);
GcsSource.Builder gcsSource = GcsSource.newBuilder();
gcsSource.addUris(gcsSourceUri);
InputConfig inputConfig = InputConfig.newBuilder().setInstancesFormat("jsonl").setGcsSource(gcsSource).build();
GcsDestination gcsDestination = GcsDestination.newBuilder().setOutputUriPrefix(gcsDestinationOutputUriPrefix).build();
OutputConfig outputConfig = OutputConfig.newBuilder().setPredictionsFormat("jsonl").setGcsDestination(gcsDestination).build();
BatchPredictionJob batchPredictionJob = BatchPredictionJob.newBuilder().setDisplayName(batchPredictionDisplayName).setModel(modelName.toString()).setModelParameters(modelParameters).setInputConfig(inputConfig).setOutputConfig(outputConfig).build();
BatchPredictionJob batchPredictionJobResponse = jobServiceClient.createBatchPredictionJob(locationName, batchPredictionJob);
System.out.println("Create Batch Prediction Job Video Object Tracking Response");
System.out.format("\tName: %s\n", batchPredictionJobResponse.getName());
System.out.format("\tDisplay Name: %s\n", batchPredictionJobResponse.getDisplayName());
System.out.format("\tModel %s\n", batchPredictionJobResponse.getModel());
System.out.format("\tModel Parameters: %s\n", batchPredictionJobResponse.getModelParameters());
System.out.format("\tState: %s\n", batchPredictionJobResponse.getState());
System.out.format("\tCreate Time: %s\n", batchPredictionJobResponse.getCreateTime());
System.out.format("\tStart Time: %s\n", batchPredictionJobResponse.getStartTime());
System.out.format("\tEnd Time: %s\n", batchPredictionJobResponse.getEndTime());
System.out.format("\tUpdate Time: %s\n", batchPredictionJobResponse.getUpdateTime());
System.out.format("\tLabels: %s\n", batchPredictionJobResponse.getLabelsMap());
InputConfig inputConfigResponse = batchPredictionJobResponse.getInputConfig();
System.out.println("\tInput Config");
System.out.format("\t\tInstances Format: %s\n", inputConfigResponse.getInstancesFormat());
GcsSource gcsSourceResponse = inputConfigResponse.getGcsSource();
System.out.println("\t\tGcs Source");
System.out.format("\t\t\tUris %s\n", gcsSourceResponse.getUrisList());
BigQuerySource bigQuerySource = inputConfigResponse.getBigquerySource();
System.out.println("\t\tBigquery Source");
System.out.format("\t\t\tInput_uri: %s\n", bigQuerySource.getInputUri());
OutputConfig outputConfigResponse = batchPredictionJobResponse.getOutputConfig();
System.out.println("\tOutput Config");
System.out.format("\t\tPredictions Format: %s\n", outputConfigResponse.getPredictionsFormat());
GcsDestination gcsDestinationResponse = outputConfigResponse.getGcsDestination();
System.out.println("\t\tGcs Destination");
System.out.format("\t\t\tOutput Uri Prefix: %s\n", gcsDestinationResponse.getOutputUriPrefix());
BigQueryDestination bigQueryDestination = outputConfigResponse.getBigqueryDestination();
System.out.println("\t\tBig Query Destination");
System.out.format("\t\t\tOutput Uri: %s\n", bigQueryDestination.getOutputUri());
BatchDedicatedResources batchDedicatedResources = batchPredictionJobResponse.getDedicatedResources();
System.out.println("\tBatch Dedicated Resources");
System.out.format("\t\tStarting Replica Count: %s\n", batchDedicatedResources.getStartingReplicaCount());
System.out.format("\t\tMax Replica Count: %s\n", batchDedicatedResources.getMaxReplicaCount());
MachineSpec machineSpec = batchDedicatedResources.getMachineSpec();
System.out.println("\t\tMachine Spec");
System.out.format("\t\t\tMachine Type: %s\n", machineSpec.getMachineType());
System.out.format("\t\t\tAccelerator Type: %s\n", machineSpec.getAcceleratorType());
System.out.format("\t\t\tAccelerator Count: %s\n", machineSpec.getAcceleratorCount());
ManualBatchTuningParameters manualBatchTuningParameters = batchPredictionJobResponse.getManualBatchTuningParameters();
System.out.println("\tManual Batch Tuning Parameters");
System.out.format("\t\tBatch Size: %s\n", manualBatchTuningParameters.getBatchSize());
OutputInfo outputInfo = batchPredictionJobResponse.getOutputInfo();
System.out.println("\tOutput Info");
System.out.format("\t\tGcs Output Directory: %s\n", outputInfo.getGcsOutputDirectory());
System.out.format("\t\tBigquery Output Dataset: %s\n", outputInfo.getBigqueryOutputDataset());
Status status = batchPredictionJobResponse.getError();
System.out.println("\tError");
System.out.format("\t\tCode: %s\n", status.getCode());
System.out.format("\t\tMessage: %s\n", status.getMessage());
List<Any> details = status.getDetailsList();
for (Status partialFailure : batchPredictionJobResponse.getPartialFailuresList()) {
System.out.println("\tPartial Failure");
System.out.format("\t\tCode: %s\n", partialFailure.getCode());
System.out.format("\t\tMessage: %s\n", partialFailure.getMessage());
List<Any> partialFailureDetailsList = partialFailure.getDetailsList();
}
ResourcesConsumed resourcesConsumed = batchPredictionJobResponse.getResourcesConsumed();
System.out.println("\tResources Consumed");
System.out.format("\t\tReplica Hours: %s\n", resourcesConsumed.getReplicaHours());
CompletionStats completionStats = batchPredictionJobResponse.getCompletionStats();
System.out.println("\tCompletion Stats");
System.out.format("\t\tSuccessful Count: %s\n", completionStats.getSuccessfulCount());
System.out.format("\t\tFailed Count: %s\n", completionStats.getFailedCount());
System.out.format("\t\tIncomplete Count: %s\n", completionStats.getIncompleteCount());
}
}
use of com.google.cloud.aiplatform.v1.MachineSpec in project java-aiplatform by googleapis.
the class DeployModelCustomTrainedModelSample method deployModelCustomTrainedModelSample.
static void deployModelCustomTrainedModelSample(String project, String endpointId, String model, String deployedModelDisplayName) throws IOException, ExecutionException, InterruptedException {
EndpointServiceSettings settings = EndpointServiceSettings.newBuilder().setEndpoint("us-central1-aiplatform.googleapis.com:443").build();
String location = "us-central1";
// the "close" method on the client to safely clean up any remaining background resources.
try (EndpointServiceClient client = EndpointServiceClient.create(settings)) {
MachineSpec machineSpec = MachineSpec.newBuilder().setMachineType("n1-standard-2").build();
DedicatedResources dedicatedResources = DedicatedResources.newBuilder().setMinReplicaCount(1).setMachineSpec(machineSpec).build();
String modelName = ModelName.of(project, location, model).toString();
DeployedModel deployedModel = DeployedModel.newBuilder().setModel(modelName).setDisplayName(deployedModelDisplayName).setDedicatedResources(dedicatedResources).build();
// key '0' assigns traffic for the newly deployed model
// Traffic percentage values must add up to 100
// Leave dictionary empty if endpoint should not accept any traffic
Map<String, Integer> trafficSplit = new HashMap<>();
trafficSplit.put("0", 100);
EndpointName endpoint = EndpointName.of(project, location, endpointId);
OperationFuture<DeployModelResponse, DeployModelOperationMetadata> response = client.deployModelAsync(endpoint, deployedModel, trafficSplit);
// You can use OperationFuture.getInitialFuture to get a future representing the initial
// response to the request, which contains information while the operation is in progress.
System.out.format("Operation name: %s\n", response.getInitialFuture().get().getName());
// OperationFuture.get() will block until the operation is finished.
DeployModelResponse deployModelResponse = response.get();
System.out.format("deployModelResponse: %s\n", deployModelResponse);
}
}
Aggregations