use of io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository in project cdap by caskdata.
the class ArtifactCacheTest method testArtifactCache.
@Test
public void testArtifactCache() throws Exception {
LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class);
ArtifactRepository artifactRepository = getInjector().getInstance(ArtifactRepository.class);
Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "some-task", "1.0.0-SNAPSHOT");
Location appJar = AppJarHelper.createDeploymentJar(locationFactory, TaskWorkerServiceTest.TestRunnableClass.class);
File appJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
Locations.linkOrCopy(appJar, appJarFile);
artifactRepository.addArtifact(artifactId, appJarFile);
CConfiguration cConf = CConfiguration.create();
String cacheDir = tmpFolder.toString();
cConf.set(Constants.ArtifactCache.LOCAL_DATA_DIR, cacheDir);
TetheringStore store = getInjector().getInstance(TetheringStore.class);
ArtifactCache cache = new ArtifactCache(cConf);
// Add a couple of tethered peers
addPeer(store, "peer1");
addPeer(store, "peer2");
RemoteClientFactory factory = new RemoteClientFactory(new NoOpDiscoveryServiceClient(getEndPoint("").toString()), new NoOpInternalAuthenticator());
HttpRequestConfig config = new DefaultHttpRequestConfig(true);
RemoteClient remoteClient = factory.createRemoteClient("", config, Constants.Gateway.INTERNAL_API_VERSION_3);
// Get artifact from first peer
File peer1ArtifactPath = cache.getArtifact(artifactId.toEntityId(), "peer1", remoteClient);
// Get the artifact again. The same path was returned
Assert.assertEquals(peer1ArtifactPath, cache.getArtifact(artifactId.toEntityId(), "peer1", remoteClient));
// Get artifact from another peer. It should be cached in a different path
File peer2ArtifactPath = cache.getArtifact(artifactId.toEntityId(), "peer2", remoteClient);
Assert.assertNotEquals(peer1ArtifactPath, peer2ArtifactPath);
// Delete and recreate the artifact to update the last modified date
artifactRepository.deleteArtifact(artifactId);
// This sleep is needed to delay the file copy so that the lastModified time on the file is different
Thread.sleep(1000);
artifactRepository.addArtifact(artifactId, appJarFile);
// Artifact should be cached in a different path
File newPeer1ArtifactPath = cache.getArtifact(artifactId.toEntityId(), "peer1", remoteClient);
Assert.assertNotEquals(peer1ArtifactPath, newPeer1ArtifactPath);
// Run the artifact cleaner
ArtifactLocalizerCleaner cleaner = new ArtifactLocalizerCleaner(Paths.get(cacheDir).resolve("peers"), 1);
cleaner.run();
// Older artifact should been deleted
Assert.assertFalse(peer1ArtifactPath.exists());
// Latest artifact should still be cached
Assert.assertTrue(newPeer1ArtifactPath.exists());
}
use of io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository in project cdap by caskdata.
the class CapabilityApplierTest method setup.
@BeforeClass
public static void setup() {
applicationLifecycleService = getInjector().getInstance(ApplicationLifecycleService.class);
capabilityWriter = getInjector().getInstance(CapabilityWriter.class);
locationFactory = getInjector().getInstance(LocationFactory.class);
artifactRepository = getInjector().getInstance(ArtifactRepository.class);
CConfiguration cConfiguration = getInjector().getInstance(CConfiguration.class);
DiscoveryServiceClient client = getInjector().getInstance(DiscoveryServiceClient.class);
RemoteClientFactory remoteClientFactory = getInjector().getInstance(RemoteClientFactory.class);
capabilityApplier = new CapabilityApplier(null, null, null, null, null, null, cConfiguration, remoteClientFactory);
}
use of io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository in project cdap by caskdata.
the class ArtifactLocalizerServiceTest method testArtifact.
@Test
public void testArtifact() throws Exception {
LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class);
ArtifactRepository artifactRepository = getInjector().getInstance(ArtifactRepository.class);
ArtifactLocalizerClient client = new ArtifactLocalizerClient(cConf, new NoOpInternalAuthenticator());
Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "some-task", "1.0.0-SNAPSHOT");
Location appJar = AppJarHelper.createDeploymentJar(locationFactory, TaskWorkerServiceTest.TestRunnableClass.class);
File appJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
File newAppJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s-copy.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
Locations.linkOrCopy(appJar, appJarFile);
appJar.delete();
artifactRepository.addArtifact(artifactId, appJarFile);
File artifactPath = client.getArtifactLocation(artifactId.toEntityId());
// Make sure the artifact was actually cached
Assert.assertTrue(artifactPath.exists());
Assert.assertTrue(artifactPath.isFile());
// Call the sidecar again and make sure the same path was returned
File sameArtifactPath = client.getArtifactLocation(artifactId.toEntityId());
Assert.assertEquals(artifactPath, sameArtifactPath);
// Delete and recreate the artifact to update the last modified date
artifactRepository.deleteArtifact(artifactId);
// This sleep is needed to delay the file copy so that the lastModified time on the file is different
Thread.sleep(1000);
// Wait a bit before recreating the artifact to make sure the last modified time is different
Files.copy(appJarFile, newAppJarFile);
artifactRepository.addArtifact(artifactId, newAppJarFile);
File newArtifactPath = client.getArtifactLocation(artifactId.toEntityId());
// Make sure the two paths arent the same and that the old one is gone
Assert.assertNotEquals(artifactPath, newArtifactPath);
Assert.assertTrue(newArtifactPath.exists());
}
use of io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository in project cdap by caskdata.
the class CapabilityApplier method autoInstallResources.
@VisibleForTesting
void autoInstallResources(String capability, @Nullable List<URL> hubs) {
ExecutorService executor = Executors.newFixedThreadPool(autoInstallThreads, Threads.createDaemonThreadFactory("installer-" + capability + "-%d"));
try {
for (URL hub : Optional.ofNullable(hubs).orElse(Collections.emptyList())) {
HubPackage[] hubPackages;
try {
URL url = new URL(hub.getProtocol(), hub.getHost(), hub.getPort(), hub.getPath() + "/packages.json");
String packagesJson = HttpClients.doGetAsString(url);
// Deserialize packages.json from hub
// See https://cdap.atlassian.net/wiki/spaces/DOCS/pages/554401840/Hub+API?src=search#Get-Hub-Catalog
hubPackages = GSON.fromJson(packagesJson, HubPackage[].class);
} catch (Exception e) {
LOG.warn("Failed to get packages.json from {} for capability {}. Ignoring error.", hub, capability, e);
continue;
}
String currentVersion = getCurrentVersion();
// Get the latest compatible version of each plugin from hub and install it
List<Future<?>> futures = Arrays.stream(hubPackages).filter(p -> p.versionIsInRange(currentVersion)).collect(Collectors.groupingBy(HubPackage::getName, Collectors.maxBy(Comparator.comparing(HubPackage::getArtifactVersion)))).values().stream().filter(Optional::isPresent).map(Optional::get).map(p -> executor.submit(() -> {
try {
LOG.debug("Installing plugins {} for capability {} from hub {}", p.getName(), capability, hub);
p.installPlugin(hub, artifactRepository, tmpDir);
} catch (Exception e) {
LOG.warn("Failed to install plugin {} for capability {} from hub {}. Ignoring error", p.getName(), capability, hub, e);
}
})).collect(Collectors.toList());
for (Future<?> future : futures) {
try {
future.get();
} catch (InterruptedException | ExecutionException e) {
LOG.warn("Ignoring error during plugin install", e);
}
}
}
} finally {
executor.shutdownNow();
}
}
Aggregations