Examples with ArtifactAsset ml.comet.experiment.artifact.ArtifactAsset used on opensource projects

Example 6 with ArtifactAsset

use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.

the class ArtifactSupportTest method testLogAndUpdateArtifact.

@Test
@Timeout(value = 300, unit = SECONDS)
void testLogAndUpdateArtifact() throws IOException {
    Path tmpDir = Files.createTempDirectory("testLogAndUpdateArtifact");
    try (OnlineExperimentImpl experiment = (OnlineExperimentImpl) createOnlineExperiment()) {
        ArtifactImpl artifact = createArtifact();
        // add local assets
        // 
        File imageFile = TestUtils.getFile(IMAGE_FILE_NAME);
        assertNotNull(imageFile);
        artifact.addAsset(imageFile, IMAGE_FILE_NAME, false, SOME_METADATA);
        File textFile = TestUtils.getFile(SOME_TEXT_FILE_NAME);
        assertNotNull(textFile);
        artifact.addAsset(textFile, SOME_TEXT_FILE_NAME, false, SOME_METADATA);
        // log artifact and check results
        // 
        CompletableFuture<LoggedArtifact> futureArtifact = experiment.logArtifact(artifact);
        LoggedArtifact loggedArtifact = futureArtifact.get(60, SECONDS);
        // download artifact and check that file was preserved
        // 
        DownloadedArtifact downloadedArtifact = loggedArtifact.download(tmpDir, AssetOverwriteStrategy.PRESERVE);
        assertNotNull(downloadedArtifact, "downloaded artifact expected");
        int originalAssetsCount = 2;
        assertEquals(originalAssetsCount, downloadedArtifact.getAssets().size(), "downloaded artifact has wrong assets size");
        // update artifact
        // 
        Set<String> newTags = new HashSet<>(Collections.singletonList("downloaded tag"));
        downloadedArtifact.setVersionTags(newTags);
        String extraAlias = "downloaded alias";
        assertTrue(downloadedArtifact.getAliases().add(extraAlias), "failed to add alias");
        String extraKey = "downloaded key";
        String extraMetaValue = "some value";
        downloadedArtifact.getMetadata().put(extraKey, extraMetaValue);
        downloadedArtifact.addAsset(Objects.requireNonNull(TestUtils.getFile(CODE_FILE_NAME)), CODE_FILE_NAME, false);
        downloadedArtifact.incrementMinorVersion();
        String newArtifactVersion = downloadedArtifact.getVersion();
        Collection<ArtifactAsset> assets = downloadedArtifact.getAssets();
        assertEquals(originalAssetsCount + 1, assets.size(), "wrong number of assets after update");
        // log downloaded artifact
        // 
        CompletableFuture<LoggedArtifact> futureUpdatedArtifact = experiment.logArtifact(downloadedArtifact);
        loggedArtifact = futureUpdatedArtifact.get(60, SECONDS);
        // read artifact from server and check that it was actually updated
        LoggedArtifact loggedArtifactFromServer = experiment.getArtifact(loggedArtifact.getName(), loggedArtifact.getWorkspace(), loggedArtifact.getVersion());
        assertEquals(newArtifactVersion, loggedArtifactFromServer.getVersion(), "wrong version");
        assertEquals(newTags, loggedArtifactFromServer.getVersionTags(), "wrong version tags");
        Set<String> expectedAliases = new HashSet<>(artifact.getAliases());
        expectedAliases.add(extraAlias);
        expectedAliases.add(ALIAS_LATEST);
        assertEquals(expectedAliases, loggedArtifactFromServer.getAliases(), "wrong aliases");
        Map<String, Object> expectedMetadata = new HashMap<>(artifact.getMetadata());
        expectedMetadata.put(extraKey, extraMetaValue);
        assertEquals(expectedMetadata, loggedArtifactFromServer.getMetadata(), "wrong metadata");
        // get assets from server and check that all assets are correct including new one
        Collection<LoggedArtifactAsset> loggedAssets = loggedArtifactFromServer.getAssets();
        assertNotNull(loggedAssets, "assets expected");
        assertEquals(assets.size(), loggedAssets.size(), "wrong assets size");
        loggedAssets.forEach(loggedArtifactAsset -> validateArtifactAsset(new ArtifactAssetImpl((LoggedArtifactAssetImpl) loggedArtifactAsset), assets));
    } catch (Throwable t) {
        fail(t);
    } finally {
        PathUtils.delete(tmpDir);
    }
}

Example 7 with ArtifactAsset

use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.

the class ArtifactImpl method appendAsset.

<T extends ArtifactAsset> void appendAsset(@NonNull final T asset) throws ConflictingArtifactAssetNameException {
    String key = asset.getLogicalPath();
    ArtifactAsset a = this.assetsMap.get(key);
    if (a != null) {
        throw new ConflictingArtifactAssetNameException(getString(CONFLICTING_ARTIFACT_ASSET_NAME, asset, key, a));
    }
    this.assetsMap.put(key, asset);
}

Example 8 with ArtifactAsset

use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.

the class BaseExperiment method downloadArtifactAsset.

/**
 * Allows to synchronously download specific {@link LoggedArtifactAsset} to the local file system.
 *
 * @param asset             the asset to be downloaded.
 * @param dir               the parent directory where asset file should be stored.
 * @param file              the relative path to the asset file.
 * @param overwriteStrategy the overwrite strategy to be applied if file already exists.
 * @return the {@link ArtifactAsset} instance with details about downloaded asset file.
 * @throws ArtifactDownloadException if failed to download asset.
 */
ArtifactAssetImpl downloadArtifactAsset(@NonNull LoggedArtifactAssetImpl asset, @NonNull Path dir, @NonNull Path file, @NonNull AssetOverwriteStrategy overwriteStrategy) throws ArtifactDownloadException {
    if (asset.isRemote()) {
        throw new ArtifactDownloadException(getString(REMOTE_ASSET_CANNOT_BE_DOWNLOADED, asset));
    }
    Path resolved;
    boolean fileAlreadyExists = false;
    try {
        Optional<Path> optionalPath = FileUtils.resolveAssetPath(dir, file, overwriteStrategy);
        if (optionalPath.isPresent()) {
            // new or overwrite
            resolved = optionalPath.get();
            if (overwriteStrategy == AssetOverwriteStrategy.OVERWRITE) {
                getLogger().warn(getString(ARTIFACT_DOWNLOAD_FILE_OVERWRITTEN, resolved, asset.getLogicalPath(), asset.artifact.getFullName()));
            }
        } else {
            // preventing original file - just warning and return FileAsset pointing to it
            resolved = dir.resolve(file);
            this.getLogger().warn(getString(ARTIFACT_ASSETS_FILE_EXISTS_PRESERVING, resolved, asset.artifact.getFullName()));
            return new ArtifactAssetImpl(asset.getLogicalPath(), resolved, Files.size(resolved), asset.getMetadata(), asset.getAssetType());
        }
    } catch (FileAlreadyExistsException e) {
        if (overwriteStrategy == AssetOverwriteStrategy.FAIL_IF_DIFFERENT) {
            try {
                resolved = Files.createTempFile(asset.getLogicalPath(), null);
                this.getLogger().debug("File '{}' already exists for asset {} and FAIL override strategy selected. " + "Start downloading to the temporary file '{}'", file, asset, resolved);
            } catch (IOException ex) {
                String msg = getString(FAILED_TO_CREATE_TEMPORARY_ASSET_DOWNLOAD_FILE, file, asset);
                this.getLogger().error(msg, ex);
                throw new ArtifactDownloadException(msg, ex);
            }
            fileAlreadyExists = true;
        } else {
            this.getLogger().error(getString(FAILED_TO_DOWNLOAD_ASSET_FILE_ALREADY_EXISTS, asset, file), e);
            throw new ArtifactDownloadException(getString(FAILED_TO_DOWNLOAD_ASSET_FILE_ALREADY_EXISTS, asset, file), e);
        }
    } catch (IOException e) {
        this.getLogger().error(getString(FAILED_TO_RESOLVE_ASSET_FILE, file, asset), e);
        throw new ArtifactDownloadException(getString(FAILED_TO_RESOLVE_ASSET_FILE, file, asset), e);
    }
    DownloadArtifactAssetOptions opts = new DownloadArtifactAssetOptions(asset.getAssetId(), asset.getArtifactVersionId(), resolved.toFile());
    RestApiResponse response = validateAndGetExperimentKey().concatMap(experimentKey -> getRestApiClient().downloadArtifactAsset(opts, experimentKey)).blockingGet();
    if (response.hasFailed()) {
        this.getLogger().error(getString(FAILED_TO_DOWNLOAD_ASSET, asset, response));
        throw new ArtifactDownloadException(getString(FAILED_TO_DOWNLOAD_ASSET, asset, response));
    }
    // this is just to mirror the Python SDK's behavior - potential performance bottleneck and system resource eater
    if (fileAlreadyExists) {
        Path assetFilePath = FileUtils.assetFilePath(dir, file);
        try {
            if (!FileUtils.fileContentsEquals(assetFilePath, resolved)) {
                this.getLogger().error(getString(FAILED_TO_DOWNLOAD_ASSET_FILE_ALREADY_EXISTS, asset, file));
                throw new ArtifactDownloadException(getString(FAILED_TO_DOWNLOAD_ASSET_FILE_ALREADY_EXISTS, asset, file));
            }
        } catch (IOException e) {
            this.getLogger().error(getString(FAILED_TO_COMPARE_CONTENT_OF_FILES, file, resolved), e);
            throw new ArtifactDownloadException(getString(FAILED_TO_COMPARE_CONTENT_OF_FILES, file, resolved), e);
        } finally {
            try {
                Files.deleteIfExists(resolved);
            } catch (IOException e) {
                this.getLogger().error(getString(FAILED_TO_DELETE_TEMPORARY_ASSET_FILE, resolved, asset), e);
            }
        }
        resolved = assetFilePath;
    }
    getLogger().info(getString(COMPLETED_DOWNLOAD_ARTIFACT_ASSET, asset.getLogicalPath(), resolved));
    try {
        return new ArtifactAssetImpl(asset.getLogicalPath(), resolved, Files.size(resolved), asset.getMetadata(), asset.getAssetType());
    } catch (IOException e) {
        this.getLogger().error(getString(FAILED_TO_READ_DOWNLOADED_FILE_SIZE, resolved), e);
        throw new ArtifactDownloadException(getString(FAILED_TO_READ_DOWNLOADED_FILE_SIZE, resolved), e);
    }
}

Example 9 with ArtifactAsset

use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.

the class BaseExperimentAsync method logArtifact.

/**
 * Asynchronously logs provided {@link Artifact}. First it synchronously upserts the artifact into the Comet
 * backend. If successful then artifact assets uploaded to the server. Finally, the artifact status committed
 * to the backend to confirm transaction status.
 *
 * @param artifact   the Comet artifact to be sent to the server.
 * @param onComplete the optional {@link Action} to be called upon operation completed,
 *                   either successful or failure.
 * @return the instance of {@link CompletableFuture} which can be used to query for {@link LoggedArtifact} with
 * details about new artifact version.
 * @throws ArtifactException if operation failed.
 */
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
CompletableFuture<LoggedArtifact> logArtifact(@NonNull final Artifact artifact, @NonNull Optional<Action> onComplete) throws ArtifactException {
    // upsert artifact
    final ArtifactEntry entry = super.upsertArtifact(artifact);
    // get new artifact's version details
    final LoggedArtifactImpl loggedArtifact = (LoggedArtifactImpl) this.getArtifactVersionDetail(Op().artifactId(entry.getArtifactId()).versionId(entry.getArtifactVersionId()).build());
    // try to log artifact assets asynchronously
    final ArtifactImpl artifactImpl = (ArtifactImpl) artifact;
    if (artifactImpl.getAssets().size() == 0) {
        getLogger().warn(getString(ARTIFACT_LOGGED_WITHOUT_ASSETS, artifactImpl.getName()));
        return CompletableFuture.completedFuture(loggedArtifact);
    }
    getLogger().info(getString(ARTIFACT_UPLOAD_STARTED, loggedArtifact.getFullName(), artifactImpl.getAssets().size()));
    CompletableFuture<LoggedArtifact> future = new CompletableFuture<>();
    // upload artifact assets
    final String artifactVersionId = entry.getArtifactVersionId();
    Stream<ArtifactAsset> assets = artifactImpl.getAssets().stream().peek(asset -> ((ArtifactAssetImpl) asset).setArtifactVersionId(artifactVersionId));
    // create parallel execution flow with errors delaying
    // allowing processing of items even if some of them failed
    AtomicInteger successfullySentCount = new AtomicInteger();
    Observable<RestApiResponse> observable = Observable.fromStream(assets).flatMap(asset -> Observable.fromSingle(this.sendArtifactAssetAsync(asset).doOnSuccess(restApiResponse -> {
        if (!restApiResponse.hasFailed()) {
            successfullySentCount.incrementAndGet();
        }
    })), true);
    if (onComplete.isPresent()) {
        observable = observable.doFinally(onComplete.get());
    }
    // subscribe to get processing results
    // noinspection ResultOfMethodCallIgnored
    observable.ignoreElements().subscribe(() -> {
        getLogger().info(getString(ARTIFACT_UPLOAD_COMPLETED, loggedArtifact.getFullName(), successfullySentCount.get()));
        // mark artifact version status as closed
        this.updateArtifactVersionState(loggedArtifact, ArtifactVersionState.CLOSED, future);
        // mark future as completed
        if (!future.isCompletedExceptionally()) {
            future.complete(loggedArtifact);
        }
    }, (throwable) -> {
        getLogger().error(getString(FAILED_TO_UPLOAD_SOME_ARTIFACT_ASSET, loggedArtifact.getFullName()), throwable);
        // mark artifact version status as failed
        this.updateArtifactVersionState(loggedArtifact, ArtifactVersionState.ERROR, future);
        // mark future as failed
        if (!future.isCompletedExceptionally()) {
            future.obtrudeException(throwable);
        }
    });
    return future;
}

Example 10 with ArtifactAsset

use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.

the class BaseExperimentAsync method sendArtifactAssetAsync.

/**
 * Attempts to send given {@link ArtifactAsset} or its subclass asynchronously.
 *
 * @param asset the artifact asset.
 * @param <T>   the type of the artifact asset.
 * @return the {@link Single} which can be used to subscribe for operation results.
 */
private <T extends ArtifactAsset> Single<RestApiResponse> sendArtifactAssetAsync(@NonNull final T asset) {
    Single<RestApiResponse> single;
    Scheduler scheduler = Schedulers.io();
    if (asset.isRemote()) {
        // remote asset
        single = validateAndGetExperimentKey().subscribeOn(scheduler).concatMap(experimentKey -> getRestApiClient().logRemoteAsset((RemoteAsset) asset, experimentKey));
    } else {
        // local asset
        single = validateAndGetExperimentKey().subscribeOn(scheduler).concatMap(experimentKey -> getRestApiClient().logAsset(asset, experimentKey));
    }
    return single.doOnSuccess(restApiResponse -> checkAndLogAssetResponse(restApiResponse, getLogger(), asset)).doOnError(throwable -> getLogger().error(getString(FAILED_TO_SEND_LOG_ARTIFACT_ASSET_REQUEST, asset), throwable));
}