use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.
the class LoggedArtifactImpl method download.
@Override
public DownloadedArtifact download(@NonNull Path folder, @NonNull AssetOverwriteStrategy overwriteStrategy) throws ArtifactException {
// create downloaded artifact
DownloadedArtifactImpl artifact = new DownloadedArtifactImpl(this);
// read all assets associated with this artifact
Collection<LoggedArtifactAsset> assets = this.getAssets();
artifact.addLoggedAssets(assets);
// check if there is assets to be downloaded
int assetsToDownload = assets.stream().filter(loggedArtifactAsset -> !loggedArtifactAsset.isRemote()).mapToInt(value -> 1).sum();
if (assetsToDownload == 0) {
// show warning and return
this.logger.warn(getString(ARTIFACT_HAS_NO_ASSETS_TO_DOWNLOAD, this.getFullName()));
return artifact;
}
this.logger.info(getString(START_DOWNLOAD_ARTIFACT_ASSETS, assetsToDownload));
// create parallel execution flow with errors delaying
// allowing processing of items even if some of them failed
Observable<ArtifactAsset> observable = Observable.fromStream(assets.stream()).filter(loggedArtifactAsset -> !loggedArtifactAsset.isRemote()).flatMap(loggedArtifactAsset -> Observable.just(loggedArtifactAsset).subscribeOn(// make it parallel on IO scheduler
Schedulers.io()).map(asset -> asset.download(folder, overwriteStrategy)), true);
// subscribe and wait for processing results
CompletableFuture<Void> result = new CompletableFuture<>();
observable.doOnNext(// update artifact asset
artifact::updateAsset).ignoreElements().blockingSubscribe(() -> {
logger.info(getString(ARTIFACT_ASSETS_DOWNLOAD_COMPLETED, this.getFullName(), assetsToDownload, folder));
result.complete(null);
}, throwable -> {
logger.error(getString(FAILED_TO_DOWNLOAD_ARTIFACT_ASSETS, this.getFullName(), folder), throwable);
result.completeExceptionally(throwable);
});
// check if any exception was raised during download and raise ArtifactException
try {
result.get();
} catch (ExecutionException ex) {
throw new ArtifactException(getString(FAILED_TO_DOWNLOAD_ARTIFACT_ASSETS, this.getFullName(), folder), ex.getCause());
} catch (InterruptedException ex) {
throw new ArtifactException(getString(FAILED_TO_DOWNLOAD_ARTIFACT_ASSETS, this.getFullName(), folder), ex);
}
return artifact;
}
use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.
the class ArtifactSupportTest method testLogAndDownloadArtifact_OVERWRITE.
@Test
@Timeout(value = 300, unit = SECONDS)
public void testLogAndDownloadArtifact_OVERWRITE() throws IOException {
Path tmpDir = Files.createTempDirectory("testLogAndDownloadArtifact_OVERWRITE");
try (OnlineExperimentImpl experiment = (OnlineExperimentImpl) createOnlineExperiment()) {
ArtifactImpl artifact = createArtifact();
// add local assets
//
artifact.addAsset(Objects.requireNonNull(TestUtils.getFile(IMAGE_FILE_NAME)), IMAGE_FILE_NAME, false, SOME_METADATA);
// log artifact
//
CompletableFuture<LoggedArtifact> futureArtifact = experiment.logArtifact(artifact);
LoggedArtifact loggedArtifact = futureArtifact.get(60, SECONDS);
// Create a conflicting file in the target directory
//
Path conflictPath = tmpDir.resolve(new File(IMAGE_FILE_NAME).toPath());
Files.write(conflictPath, "some data".getBytes(StandardCharsets.UTF_8));
// download artifact and check that file was overwritten
//
DownloadedArtifact downloadedArtifact = loggedArtifact.download(tmpDir, AssetOverwriteStrategy.OVERWRITE);
assertNotNull(downloadedArtifact, "downloaded artifact expected");
Collection<ArtifactAsset> assets = downloadedArtifact.getAssets();
assertEquals(1, assets.size());
ArtifactAsset asset = assets.iterator().next();
Path assetFile = tmpDir.resolve(asset.getLogicalPath());
assertTrue(PathUtils.fileContentEquals(Objects.requireNonNull(TestUtils.getFile(IMAGE_FILE_NAME)).toPath(), assetFile));
} catch (Throwable t) {
fail(t);
} finally {
PathUtils.delete(tmpDir);
}
}
use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.
the class ArtifactSupportTest method testLogAndGetArtifact.
@Test
public void testLogAndGetArtifact() {
try (OnlineExperimentImpl experiment = (OnlineExperimentImpl) createOnlineExperiment()) {
ArtifactImpl artifact = createArtifact();
// add remote assets
//
URI firstAssetLink = new URI("s3://bucket/folder/firstAssetFile.extension");
String firstAssetFileName = "firstAssetFileName";
artifact.addRemoteAsset(firstAssetLink, firstAssetFileName);
String secondAssetExpectedFileName = "secondAssetFile.extension";
URI secondAssetLink = new URI("s3://bucket/folder/" + secondAssetExpectedFileName);
artifact.addRemoteAsset(secondAssetLink, secondAssetExpectedFileName);
// add local assets
//
artifact.addAsset(Objects.requireNonNull(TestUtils.getFile(IMAGE_FILE_NAME)), IMAGE_FILE_NAME, false, SOME_METADATA);
artifact.addAsset(Objects.requireNonNull(TestUtils.getFile(CODE_FILE_NAME)), CODE_FILE_NAME, false);
byte[] someData = "some data".getBytes(StandardCharsets.UTF_8);
String someDataName = "someDataName";
artifact.addAsset(someData, someDataName);
// add assets folder
//
artifact.addAssetFolder(assetsFolder.toFile(), true, true);
// the logged artifact validator
Function4<LoggedArtifact, ArtifactImpl, String, List<String>, Void> loggedArtifactValidator = (actual, original, experimentKey, expectedAliases) -> {
assertNotNull(actual, "logged artifact expected");
assertEquals(original.getType(), actual.getArtifactType(), "wrong artifact type");
assertEquals(new HashSet<>(expectedAliases), actual.getAliases(), "wrong aliases");
assertEquals(SOME_METADATA, actual.getMetadata(), "wrong metadata");
assertEquals(new HashSet<>(original.getVersionTags()), actual.getVersionTags(), "wrong version tags");
assertEquals(WORKSPACE_NAME, actual.getWorkspace(), "wrong workspace");
assertEquals(experimentKey, actual.getSourceExperimentKey(), "wrong experiment key");
assertEquals(original.getName(), actual.getName(), "wrong artifact name");
return null;
};
// check artifacts-in-progress counter before
assertEquals(0, experiment.getArtifactsInProgress().get(), "artifacts-in-progress counter must be zero at start");
// log artifact and check results
//
CompletableFuture<LoggedArtifact> futureArtifact = experiment.logArtifact(artifact);
// check artifacts-in-progress counter while in progress
assertEquals(1, experiment.getArtifactsInProgress().get(), "artifacts-in-progress counter has wrong value while still in progress");
LoggedArtifact loggedArtifact = futureArtifact.get(60, SECONDS);
// check artifacts-in-progress counter after
Awaitility.await("artifacts-in-progress counter must be decreased").pollInterval(10, TimeUnit.MILLISECONDS).atMost(1, TimeUnit.SECONDS).until(() -> experiment.getArtifactsInProgress().get() == 0);
assertEquals(0, experiment.getArtifactsInProgress().get(), "artifacts-in-progress counter must be zero after log operation completed");
List<String> expectedAliases = new ArrayList<>(artifact.getAliases());
loggedArtifactValidator.apply(loggedArtifact, artifact, experiment.getExperimentKey(), expectedAliases);
// get artifact details from server and check its correctness
//
LoggedArtifact loggedArtifactFromServer = experiment.getArtifact(loggedArtifact.getName(), loggedArtifact.getWorkspace(), loggedArtifact.getVersion());
// added by the backend automatically
expectedAliases.add(ALIAS_LATEST);
loggedArtifactValidator.apply(loggedArtifactFromServer, artifact, experiment.getExperimentKey(), expectedAliases);
// check that correct assets was logged
//
Collection<LoggedArtifactAsset> loggedAssets = loggedArtifactFromServer.getAssets();
Collection<ArtifactAsset> assets = artifact.getAssets();
assertEquals(assets.size(), loggedAssets.size(), "wrong size");
loggedAssets.forEach(loggedArtifactAsset -> validateArtifactAsset(new ArtifactAssetImpl((LoggedArtifactAssetImpl) loggedArtifactAsset), assets));
} catch (Throwable t) {
fail(t);
}
}
use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.
the class ArtifactSupportTest method validateArtifactAsset.
static void validateArtifactAsset(ArtifactAsset artifactAsset, Collection<ArtifactAsset> assets) {
AtomicBoolean matchFound = new AtomicBoolean(false);
assets.stream().filter(asset -> Objects.equals(asset.getLogicalPath(), artifactAsset.getLogicalPath())).forEach(asset -> {
matchFound.set(true);
if (asset.isRemote()) {
assertTrue(artifactAsset.isRemote());
assertTrue(artifactAsset.getLink().isPresent(), "remote link expected in logged asset");
assertTrue(asset.getLink().isPresent(), "remote link expected in asset");
assertEquals(asset.getLink().get(), artifactAsset.getLink().get(), "wrong URI");
} else {
assertFalse(artifactAsset.isRemote());
}
if (asset.getMetadata() != null) {
assertEquals(asset.getMetadata(), artifactAsset.getMetadata(), "wrong metadata");
} else {
assertEquals(0, artifactAsset.getMetadata().size(), "empty metadata expected");
}
if (Objects.equals(asset.getType(), ASSET.type())) {
assertEquals(UNKNOWN.type(), artifactAsset.getType());
} else {
assertEquals(asset.getType(), artifactAsset.getType(), "wrong asset type");
}
});
assertTrue(matchFound.get(), String.format("no match found for %s", artifactAsset));
}
use of ml.comet.experiment.artifact.ArtifactAsset in project comet-java-sdk by comet-ml.
the class ArtifactSupportTest method testLogAndDownloadArtifact_PRESERVE.
@Test
@Timeout(value = 300, unit = SECONDS)
public void testLogAndDownloadArtifact_PRESERVE() throws IOException {
Path tmpDir = Files.createTempDirectory("testLogAndDownloadArtifact_PREVENT");
try (OnlineExperimentImpl experiment = (OnlineExperimentImpl) createOnlineExperiment()) {
ArtifactImpl artifact = createArtifact();
// add local assets
//
artifact.addAsset(Objects.requireNonNull(TestUtils.getFile(IMAGE_FILE_NAME)), IMAGE_FILE_NAME, false, SOME_METADATA);
// log artifact
//
CompletableFuture<LoggedArtifact> futureArtifact = experiment.logArtifact(artifact);
LoggedArtifact loggedArtifact = futureArtifact.get(60, SECONDS);
// Create a conflicting file in the target directory
//
Path conflictPath = tmpDir.resolve(new File(IMAGE_FILE_NAME).toPath());
byte[] someData = "some data".getBytes(StandardCharsets.UTF_8);
Files.write(conflictPath, someData);
// download artifact and check that file was preserved
//
DownloadedArtifact downloadedArtifact = loggedArtifact.download(tmpDir, AssetOverwriteStrategy.PRESERVE);
assertNotNull(downloadedArtifact, "downloaded artifact expected");
Collection<ArtifactAsset> assets = downloadedArtifact.getAssets();
assertEquals(1, assets.size());
byte[] fileData = Files.readAllBytes(conflictPath);
assertArrayEquals(someData, fileData);
} catch (Throwable t) {
fail(t);
} finally {
PathUtils.delete(tmpDir);
}
}
Aggregations