Search in sources :

Example 1 with RemoteAsset

use of ml.comet.experiment.asset.RemoteAsset in project comet-java-sdk by comet-ml.

the class AssetUtilsTest method testCreateRemoteAsset_fileNameSelection.

@ParameterizedTest
@CsvSource({ "s3://bucket/folder/file.extension, someFileName, someFileName", "s3://bucket/folder/file.extension,, file.extension", "s3://bucket,, " + REMOTE_FILE_NAME_DEFAULT })
public void testCreateRemoteAsset_fileNameSelection(URI uri, String fileName, String expectedFileName) {
    RemoteAsset asset = createRemoteAsset(uri, ofNullable(fileName), false, empty(), empty());
    assertNotNull(asset);
    assertEquals(expectedFileName, asset.getLogicalPath());
}
Also used : AssetUtils.createRemoteAsset(ml.comet.experiment.impl.utils.AssetUtils.createRemoteAsset) RemoteAsset(ml.comet.experiment.asset.RemoteAsset) CsvSource(org.junit.jupiter.params.provider.CsvSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 2 with RemoteAsset

use of ml.comet.experiment.asset.RemoteAsset in project comet-java-sdk by comet-ml.

the class BaseExperimentAsync method sendArtifactAssetAsync.

/**
 * Attempts to send given {@link ArtifactAsset} or its subclass asynchronously.
 *
 * @param asset the artifact asset.
 * @param <T>   the type of the artifact asset.
 * @return the {@link Single} which can be used to subscribe for operation results.
 */
private <T extends ArtifactAsset> Single<RestApiResponse> sendArtifactAssetAsync(@NonNull final T asset) {
    Single<RestApiResponse> single;
    Scheduler scheduler = Schedulers.io();
    if (asset.isRemote()) {
        // remote asset
        single = validateAndGetExperimentKey().subscribeOn(scheduler).concatMap(experimentKey -> getRestApiClient().logRemoteAsset((RemoteAsset) asset, experimentKey));
    } else {
        // local asset
        single = validateAndGetExperimentKey().subscribeOn(scheduler).concatMap(experimentKey -> getRestApiClient().logAsset(asset, experimentKey));
    }
    return single.doOnSuccess(restApiResponse -> checkAndLogAssetResponse(restApiResponse, getLogger(), asset)).doOnError(throwable -> getLogger().error(getString(FAILED_TO_SEND_LOG_ARTIFACT_ASSET_REQUEST, asset), throwable));
}
Also used : RestApiUtils.createLogParamRequest(ml.comet.experiment.impl.utils.RestApiUtils.createLogParamRequest) RestApiUtils.createLogStartTimeRequest(ml.comet.experiment.impl.utils.RestApiUtils.createLogStartTimeRequest) AssetImpl(ml.comet.experiment.impl.asset.AssetImpl) RemoteAssetImpl(ml.comet.experiment.impl.asset.RemoteAssetImpl) FAILED_TO_FINALIZE_ARTIFACT_VERSION(ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_FINALIZE_ARTIFACT_VERSION) RestApiUtils.createLogEndTimeRequest(ml.comet.experiment.impl.utils.RestApiUtils.createLogEndTimeRequest) StringUtils(org.apache.commons.lang3.StringUtils) LoggedArtifact(ml.comet.experiment.artifact.LoggedArtifact) ArtifactException(ml.comet.experiment.artifact.ArtifactException) RemoteAsset(ml.comet.experiment.asset.RemoteAsset) OutputUpdate(ml.comet.experiment.impl.rest.OutputUpdate) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RestApiUtils.createGraphRequest(ml.comet.experiment.impl.utils.RestApiUtils.createGraphRequest) Duration(java.time.Duration) Map(java.util.Map) ARTIFACT_UPLOAD_STARTED(ml.comet.experiment.impl.resources.LogMessages.ARTIFACT_UPLOAD_STARTED) RestApiUtils.createLogLineRequest(ml.comet.experiment.impl.utils.RestApiUtils.createLogLineRequest) RestApiUtils.createGitMetadataRequest(ml.comet.experiment.impl.utils.RestApiUtils.createGitMetadataRequest) URI(java.net.URI) RestApiUtils.createLogMetricRequest(ml.comet.experiment.impl.utils.RestApiUtils.createLogMetricRequest) ArtifactAsset(ml.comet.experiment.artifact.ArtifactAsset) LogMessages.getString(ml.comet.experiment.impl.resources.LogMessages.getString) FAILED_TO_SEND_LOG_REQUEST(ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_SEND_LOG_REQUEST) FAILED_TO_LOG_ASSET_FOLDER(ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_LOG_ASSET_FOLDER) FAILED_TO_UPLOAD_SOME_ARTIFACT_ASSET(ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_UPLOAD_SOME_ARTIFACT_ASSET) NonNull(lombok.NonNull) AssetUtils.createAssetFromData(ml.comet.experiment.impl.utils.AssetUtils.createAssetFromData) RestApiUtils.createLogHtmlRequest(ml.comet.experiment.impl.utils.RestApiUtils.createLogHtmlRequest) Action(io.reactivex.rxjava3.functions.Action) HtmlRest(ml.comet.experiment.impl.rest.HtmlRest) AssetType(ml.comet.experiment.impl.asset.AssetType) Objects(java.util.Objects) Stream(java.util.stream.Stream) Asset(ml.comet.experiment.asset.Asset) BiFunction(io.reactivex.rxjava3.functions.BiFunction) AssetUtils(ml.comet.experiment.impl.utils.AssetUtils) Optional(java.util.Optional) FAILED_TO_SEND_LOG_ASSET_REQUEST(ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_SEND_LOG_ASSET_REQUEST) Optional.empty(java.util.Optional.empty) Single(io.reactivex.rxjava3.core.Single) ASSETS_FOLDER_UPLOAD_COMPLETED(ml.comet.experiment.impl.resources.LogMessages.ASSETS_FOLDER_UPLOAD_COMPLETED) CompletableFuture(java.util.concurrent.CompletableFuture) ArtifactAssetImpl(ml.comet.experiment.impl.asset.ArtifactAssetImpl) Schedulers(io.reactivex.rxjava3.schedulers.Schedulers) MetricRest(ml.comet.experiment.impl.rest.MetricRest) RestApiUtils.createTagRequest(ml.comet.experiment.impl.utils.RestApiUtils.createTagRequest) ParameterRest(ml.comet.experiment.impl.rest.ParameterRest) Artifact(ml.comet.experiment.artifact.Artifact) Observable(io.reactivex.rxjava3.core.Observable) Scheduler(io.reactivex.rxjava3.core.Scheduler) LogOtherRest(ml.comet.experiment.impl.rest.LogOtherRest) ArtifactVersionState(ml.comet.experiment.impl.rest.ArtifactVersionState) RestApiUtils.createLogOtherRequest(ml.comet.experiment.impl.utils.RestApiUtils.createLogOtherRequest) GitMetaData(ml.comet.experiment.model.GitMetaData) RestApiResponse(ml.comet.experiment.impl.rest.RestApiResponse) Logger(org.slf4j.Logger) ARTIFACT_LOGGED_WITHOUT_ASSETS(ml.comet.experiment.impl.resources.LogMessages.ARTIFACT_LOGGED_WITHOUT_ASSETS) ExperimentContext(ml.comet.experiment.context.ExperimentContext) File(java.io.File) ARTIFACT_UPLOAD_COMPLETED(ml.comet.experiment.impl.resources.LogMessages.ARTIFACT_UPLOAD_COMPLETED) LOG_REMOTE_ASSET_URI_FILE_NAME_TO_DEFAULT(ml.comet.experiment.impl.resources.LogMessages.LOG_REMOTE_ASSET_URI_FILE_NAME_TO_DEFAULT) FAILED_TO_SEND_LOG_ARTIFACT_ASSET_REQUEST(ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_SEND_LOG_ARTIFACT_ASSET_REQUEST) AssetUtils.createAssetFromFile(ml.comet.experiment.impl.utils.AssetUtils.createAssetFromFile) LOG_ASSET_FOLDER_EMPTY(ml.comet.experiment.impl.resources.LogMessages.LOG_ASSET_FOLDER_EMPTY) FAILED_TO_LOG_SOME_ASSET_FROM_FOLDER(ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_LOG_SOME_ASSET_FROM_FOLDER) ArtifactEntry(ml.comet.experiment.impl.rest.ArtifactEntry) Op(ml.comet.experiment.artifact.GetArtifactOptions.Op) Scheduler(io.reactivex.rxjava3.core.Scheduler) RestApiResponse(ml.comet.experiment.impl.rest.RestApiResponse)

Example 3 with RemoteAsset

use of ml.comet.experiment.asset.RemoteAsset in project comet-java-sdk by comet-ml.

the class AssetUtilsTest method testCreateRemoteAsset_correctTypeCheck.

@Test
public void testCreateRemoteAsset_correctTypeCheck() throws URISyntaxException {
    URI uri = new URI("s3://bucket/folder/someFile");
    RemoteAsset asset = createRemoteAsset(uri, empty(), false, empty(), empty());
    assertNotNull(asset);
    assertEquals(ASSET.type(), asset.getType());
    String expected = NOTEBOOK.type();
    asset = createRemoteAsset(uri, empty(), false, empty(), Optional.of(expected));
    assertNotNull(asset);
    assertEquals(expected, asset.getType());
}
Also used : AssetUtils.createRemoteAsset(ml.comet.experiment.impl.utils.AssetUtils.createRemoteAsset) RemoteAsset(ml.comet.experiment.asset.RemoteAsset) URI(java.net.URI) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

RemoteAsset (ml.comet.experiment.asset.RemoteAsset)3 URI (java.net.URI)2 Observable (io.reactivex.rxjava3.core.Observable)1 Scheduler (io.reactivex.rxjava3.core.Scheduler)1 Single (io.reactivex.rxjava3.core.Single)1 Action (io.reactivex.rxjava3.functions.Action)1 BiFunction (io.reactivex.rxjava3.functions.BiFunction)1 Schedulers (io.reactivex.rxjava3.schedulers.Schedulers)1 File (java.io.File)1 Duration (java.time.Duration)1 Map (java.util.Map)1 Objects (java.util.Objects)1 Optional (java.util.Optional)1 Optional.empty (java.util.Optional.empty)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1 Stream (java.util.stream.Stream)1 NonNull (lombok.NonNull)1 Artifact (ml.comet.experiment.artifact.Artifact)1 ArtifactAsset (ml.comet.experiment.artifact.ArtifactAsset)1