Search in sources :

Example 11 with ArtifactRepository

use of io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository in project cdap by caskdata.

the class ArtifactCacheTest method testArtifactCache.

@Test
public void testArtifactCache() throws Exception {
    LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class);
    ArtifactRepository artifactRepository = getInjector().getInstance(ArtifactRepository.class);
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "some-task", "1.0.0-SNAPSHOT");
    Location appJar = AppJarHelper.createDeploymentJar(locationFactory, TaskWorkerServiceTest.TestRunnableClass.class);
    File appJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
    Locations.linkOrCopy(appJar, appJarFile);
    artifactRepository.addArtifact(artifactId, appJarFile);
    CConfiguration cConf = CConfiguration.create();
    String cacheDir = tmpFolder.toString();
    cConf.set(Constants.ArtifactCache.LOCAL_DATA_DIR, cacheDir);
    TetheringStore store = getInjector().getInstance(TetheringStore.class);
    ArtifactCache cache = new ArtifactCache(cConf);
    // Add a couple of tethered peers
    addPeer(store, "peer1");
    addPeer(store, "peer2");
    RemoteClientFactory factory = new RemoteClientFactory(new NoOpDiscoveryServiceClient(getEndPoint("").toString()), new NoOpInternalAuthenticator());
    HttpRequestConfig config = new DefaultHttpRequestConfig(true);
    RemoteClient remoteClient = factory.createRemoteClient("", config, Constants.Gateway.INTERNAL_API_VERSION_3);
    // Get artifact from first peer
    File peer1ArtifactPath = cache.getArtifact(artifactId.toEntityId(), "peer1", remoteClient);
    // Get the artifact again. The same path was returned
    Assert.assertEquals(peer1ArtifactPath, cache.getArtifact(artifactId.toEntityId(), "peer1", remoteClient));
    // Get artifact from another peer. It should be cached in a different path
    File peer2ArtifactPath = cache.getArtifact(artifactId.toEntityId(), "peer2", remoteClient);
    Assert.assertNotEquals(peer1ArtifactPath, peer2ArtifactPath);
    // Delete and recreate the artifact to update the last modified date
    artifactRepository.deleteArtifact(artifactId);
    // This sleep is needed to delay the file copy so that the lastModified time on the file is different
    Thread.sleep(1000);
    artifactRepository.addArtifact(artifactId, appJarFile);
    // Artifact should be cached in a different path
    File newPeer1ArtifactPath = cache.getArtifact(artifactId.toEntityId(), "peer1", remoteClient);
    Assert.assertNotEquals(peer1ArtifactPath, newPeer1ArtifactPath);
    // Run the artifact cleaner
    ArtifactLocalizerCleaner cleaner = new ArtifactLocalizerCleaner(Paths.get(cacheDir).resolve("peers"), 1);
    cleaner.run();
    // Older artifact should been deleted
    Assert.assertFalse(peer1ArtifactPath.exists());
    // Latest artifact should still be cached
    Assert.assertTrue(newPeer1ArtifactPath.exists());
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) DefaultHttpRequestConfig(io.cdap.cdap.common.http.DefaultHttpRequestConfig) NoOpInternalAuthenticator(io.cdap.cdap.common.internal.remote.NoOpInternalAuthenticator) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) TaskWorkerServiceTest(io.cdap.cdap.internal.app.worker.TaskWorkerServiceTest) DefaultHttpRequestConfig(io.cdap.cdap.common.http.DefaultHttpRequestConfig) HttpRequestConfig(io.cdap.common.http.HttpRequestConfig) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) LocationFactory(org.apache.twill.filesystem.LocationFactory) ArtifactLocalizerCleaner(io.cdap.cdap.internal.app.worker.sidecar.ArtifactLocalizerCleaner) RemoteClient(io.cdap.cdap.common.internal.remote.RemoteClient) Id(io.cdap.cdap.common.id.Id) File(java.io.File) Location(org.apache.twill.filesystem.Location) Test(org.junit.Test) TaskWorkerServiceTest(io.cdap.cdap.internal.app.worker.TaskWorkerServiceTest)

Example 12 with ArtifactRepository

use of io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository in project cdap by caskdata.

the class CapabilityApplierTest method setup.

@BeforeClass
public static void setup() {
    applicationLifecycleService = getInjector().getInstance(ApplicationLifecycleService.class);
    capabilityWriter = getInjector().getInstance(CapabilityWriter.class);
    locationFactory = getInjector().getInstance(LocationFactory.class);
    artifactRepository = getInjector().getInstance(ArtifactRepository.class);
    CConfiguration cConfiguration = getInjector().getInstance(CConfiguration.class);
    DiscoveryServiceClient client = getInjector().getInstance(DiscoveryServiceClient.class);
    RemoteClientFactory remoteClientFactory = getInjector().getInstance(RemoteClientFactory.class);
    capabilityApplier = new CapabilityApplier(null, null, null, null, null, null, cConfiguration, remoteClientFactory);
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) ApplicationLifecycleService(io.cdap.cdap.internal.app.services.ApplicationLifecycleService) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) LocationFactory(org.apache.twill.filesystem.LocationFactory) BeforeClass(org.junit.BeforeClass)

Example 13 with ArtifactRepository

use of io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository in project cdap by caskdata.

the class ArtifactLocalizerServiceTest method testArtifact.

@Test
public void testArtifact() throws Exception {
    LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class);
    ArtifactRepository artifactRepository = getInjector().getInstance(ArtifactRepository.class);
    ArtifactLocalizerClient client = new ArtifactLocalizerClient(cConf, new NoOpInternalAuthenticator());
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "some-task", "1.0.0-SNAPSHOT");
    Location appJar = AppJarHelper.createDeploymentJar(locationFactory, TaskWorkerServiceTest.TestRunnableClass.class);
    File appJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
    File newAppJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s-copy.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
    Locations.linkOrCopy(appJar, appJarFile);
    appJar.delete();
    artifactRepository.addArtifact(artifactId, appJarFile);
    File artifactPath = client.getArtifactLocation(artifactId.toEntityId());
    // Make sure the artifact was actually cached
    Assert.assertTrue(artifactPath.exists());
    Assert.assertTrue(artifactPath.isFile());
    // Call the sidecar again and make sure the same path was returned
    File sameArtifactPath = client.getArtifactLocation(artifactId.toEntityId());
    Assert.assertEquals(artifactPath, sameArtifactPath);
    // Delete and recreate the artifact to update the last modified date
    artifactRepository.deleteArtifact(artifactId);
    // This sleep is needed to delay the file copy so that the lastModified time on the file is different
    Thread.sleep(1000);
    // Wait a bit before recreating the artifact to make sure the last modified time is different
    Files.copy(appJarFile, newAppJarFile);
    artifactRepository.addArtifact(artifactId, newAppJarFile);
    File newArtifactPath = client.getArtifactLocation(artifactId.toEntityId());
    // Make sure the two paths arent the same and that the old one is gone
    Assert.assertNotEquals(artifactPath, newArtifactPath);
    Assert.assertTrue(newArtifactPath.exists());
}
Also used : NoOpInternalAuthenticator(io.cdap.cdap.common.internal.remote.NoOpInternalAuthenticator) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) TaskWorkerServiceTest(io.cdap.cdap.internal.app.worker.TaskWorkerServiceTest) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) File(java.io.File) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) Location(org.apache.twill.filesystem.Location) Test(org.junit.Test) TaskWorkerServiceTest(io.cdap.cdap.internal.app.worker.TaskWorkerServiceTest)

Example 14 with ArtifactRepository

use of io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository in project cdap by caskdata.

the class CapabilityApplier method autoInstallResources.

@VisibleForTesting
void autoInstallResources(String capability, @Nullable List<URL> hubs) {
    ExecutorService executor = Executors.newFixedThreadPool(autoInstallThreads, Threads.createDaemonThreadFactory("installer-" + capability + "-%d"));
    try {
        for (URL hub : Optional.ofNullable(hubs).orElse(Collections.emptyList())) {
            HubPackage[] hubPackages;
            try {
                URL url = new URL(hub.getProtocol(), hub.getHost(), hub.getPort(), hub.getPath() + "/packages.json");
                String packagesJson = HttpClients.doGetAsString(url);
                // Deserialize packages.json from hub
                // See https://cdap.atlassian.net/wiki/spaces/DOCS/pages/554401840/Hub+API?src=search#Get-Hub-Catalog
                hubPackages = GSON.fromJson(packagesJson, HubPackage[].class);
            } catch (Exception e) {
                LOG.warn("Failed to get packages.json from {} for capability {}. Ignoring error.", hub, capability, e);
                continue;
            }
            String currentVersion = getCurrentVersion();
            // Get the latest compatible version of each plugin from hub and install it
            List<Future<?>> futures = Arrays.stream(hubPackages).filter(p -> p.versionIsInRange(currentVersion)).collect(Collectors.groupingBy(HubPackage::getName, Collectors.maxBy(Comparator.comparing(HubPackage::getArtifactVersion)))).values().stream().filter(Optional::isPresent).map(Optional::get).map(p -> executor.submit(() -> {
                try {
                    LOG.debug("Installing plugins {} for capability {} from hub {}", p.getName(), capability, hub);
                    p.installPlugin(hub, artifactRepository, tmpDir);
                } catch (Exception e) {
                    LOG.warn("Failed to install plugin {} for capability {} from hub {}. Ignoring error", p.getName(), capability, hub, e);
                }
            })).collect(Collectors.toList());
            for (Future<?> future : futures) {
                try {
                    future.get();
                } catch (InterruptedException | ExecutionException e) {
                    LOG.warn("Ignoring error during plugin install", e);
                }
            }
        }
    } finally {
        executor.shutdownNow();
    }
}
Also used : MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) ApplicationDetail(io.cdap.cdap.proto.ApplicationDetail) Arrays(java.util.Arrays) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) URL(java.net.URL) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) Future(java.util.concurrent.Future) Gson(com.google.gson.Gson) ArtifactSummary(io.cdap.cdap.api.artifact.ArtifactSummary) ArtifactSortOrder(io.cdap.cdap.proto.artifact.ArtifactSortOrder) Map(java.util.Map) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) ApplicationNotFoundException(io.cdap.cdap.common.ApplicationNotFoundException) ProgramTerminator(io.cdap.cdap.internal.app.deploy.ProgramTerminator) Threads(org.apache.twill.common.Threads) EntityResult(io.cdap.cdap.internal.entity.EntityResult) Collection(java.util.Collection) Set(java.util.Set) ApplicationLifecycleService(io.cdap.cdap.internal.app.services.ApplicationLifecycleService) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Objects(java.util.Objects) NamespaceMeta(io.cdap.cdap.proto.NamespaceMeta) List(java.util.List) HubPackage(io.cdap.cdap.internal.capability.autoinstall.HubPackage) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) Optional(java.util.Optional) Constants(io.cdap.cdap.common.conf.Constants) InvalidArtifactException(io.cdap.cdap.common.InvalidArtifactException) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) NotFoundException(io.cdap.cdap.common.NotFoundException) RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) Retries(io.cdap.cdap.common.service.Retries) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) VersionHelper(io.cdap.cdap.gateway.handlers.util.VersionHelper) Arguments(io.cdap.cdap.app.runtime.Arguments) UnauthorizedException(io.cdap.cdap.security.spi.authorization.UnauthorizedException) HashMap(java.util.HashMap) ProgramType(io.cdap.cdap.proto.ProgramType) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ArtifactVersionRange(io.cdap.cdap.api.artifact.ArtifactVersionRange) ArtifactDetail(io.cdap.cdap.internal.app.runtime.artifact.ArtifactDetail) ExecutorService(java.util.concurrent.ExecutorService) Nullable(javax.annotation.Nullable) ProgramLifecycleService(io.cdap.cdap.internal.app.services.ProgramLifecycleService) Logger(org.slf4j.Logger) ProgramId(io.cdap.cdap.proto.id.ProgramId) IOException(java.io.IOException) File(java.io.File) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) NamespaceAdmin(io.cdap.cdap.common.namespace.NamespaceAdmin) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) SystemProgramManagementService(io.cdap.cdap.internal.app.services.SystemProgramManagementService) Collections(java.util.Collections) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) Optional(java.util.Optional) URL(java.net.URL) HubPackage(io.cdap.cdap.internal.capability.autoinstall.HubPackage) ApplicationNotFoundException(io.cdap.cdap.common.ApplicationNotFoundException) InvalidArtifactException(io.cdap.cdap.common.InvalidArtifactException) NotFoundException(io.cdap.cdap.common.NotFoundException) UnauthorizedException(io.cdap.cdap.security.spi.authorization.UnauthorizedException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) ExecutorService(java.util.concurrent.ExecutorService) Future(java.util.concurrent.Future) ExecutionException(java.util.concurrent.ExecutionException) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

ArtifactRepository (io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository)14 File (java.io.File)9 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)7 Id (io.cdap.cdap.common.id.Id)7 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)7 LocationFactory (org.apache.twill.filesystem.LocationFactory)7 Location (org.apache.twill.filesystem.Location)6 Test (org.junit.Test)6 LocalLocationFactory (org.apache.twill.filesystem.LocalLocationFactory)5 NoOpInternalAuthenticator (io.cdap.cdap.common.internal.remote.NoOpInternalAuthenticator)4 RemoteClientFactory (io.cdap.cdap.common.internal.remote.RemoteClientFactory)4 Gson (com.google.gson.Gson)3 ApplicationClass (io.cdap.cdap.api.artifact.ApplicationClass)3 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)3 TaskWorkerServiceTest (io.cdap.cdap.internal.app.worker.TaskWorkerServiceTest)3 Injector (com.google.inject.Injector)2 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)2 ArtifactRange (io.cdap.cdap.api.artifact.ArtifactRange)2 ConfigResponse (io.cdap.cdap.app.deploy.ConfigResponse)2 Configurator (io.cdap.cdap.app.deploy.Configurator)2