Search in sources :

Example 1 with HubPackage

use of io.cdap.cdap.internal.capability.autoinstall.HubPackage in project cdap by caskdata.

the class AutoInstallTest method testAutoInstallPlugins.

@Test
public void testAutoInstallPlugins() throws Exception {
    // Setup mocks
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.setInt(Constants.Capability.AUTO_INSTALL_THREADS, 5);
    ArtifactRepository artifactRepository = PowerMockito.mock(ArtifactRepository.class);
    RemoteClientFactory remoteClientFactory = new RemoteClientFactory(null, new NoOpInternalAuthenticator());
    CapabilityApplier capabilityApplier = new CapabilityApplier(null, null, null, null, null, artifactRepository, cConf, remoteClientFactory);
    CapabilityApplier ca = Mockito.spy(capabilityApplier);
    PowerMockito.mockStatic(HttpClients.class);
    PowerMockito.mockStatic(Files.class);
    PowerMockito.mockStatic(File.class);
    PowerMockito.mockStatic(Paths.class);
    PowerMockito.mockStatic(java.nio.file.Files.class);
    File mockFile = TEMP_FOLDER.newFile();
    Mockito.when(File.createTempFile(anyString(), anyString(), any())).thenReturn(mockFile);
    Path mockPath = PowerMockito.mock(Path.class);
    Mockito.when(Paths.get(mockFile.getPath())).thenReturn(mockPath);
    URL packagesUrl = new URL("https://my.hub.io/packages.json");
    HubPackage pkg1 = new HubPackage("my-plugin", "1.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.1.1,6.3.0]", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    HubPackage pkg2 = new HubPackage("my-plugin", "2.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.3.1,6.4.0]", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    HubPackage pkg3 = new HubPackage("my-plugin", "3.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.4.1,7.0.0-SNAPSHOT)", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    String packagesJson = GSON.toJson(ImmutableList.of(pkg1, pkg2, pkg3));
    URL specUrl = new URL("https://my.hub.io/packages/my-plugin/2.0.0/spec.json");
    List<Spec.Action.Argument> arguments = Arrays.asList(new Spec.Action.Argument("config", "my-plugin-2.0.0.json", false), new Spec.Action.Argument("jar", "my-plugin-2.0.0.jar", false), new Spec.Action.Argument("name", "my-plugin", false), new Spec.Action.Argument("version", "2.0.0", false));
    Spec.Action action = new Spec.Action("one_step_deploy_plugin", "Deploy my plugin", arguments);
    Spec spec = new Spec("1.0", "My Plugin", "My Plugin", "Cask", "Cask", 1554766945, "[6.3.1,6.4.0]", Collections.singletonList("hydrator-plugin"), true, Collections.singletonList(action));
    String specJson = GSON.toJson(spec);
    URL configUrl = new URL("https://my.hub.io/packages/my-plugin/2.0.0/my-plugin-2.0.0.json");
    Map<String, String> properties = ImmutableMap.of("key1", "value1", "key2", "value2");
    Map<String, Object> config = ImmutableMap.of("parents", ImmutableList.of("system:cdap-data-pipeline[6.3.1,6.4.0]", "system:cdap-data-streams[6.3.1,6.4.0]"), "properties", properties);
    String configJson = GSON.toJson(config);
    // Mock http requests to hub
    Mockito.when(HttpClients.doGetAsString(packagesUrl)).thenReturn(packagesJson);
    Mockito.when(HttpClients.doGetAsString(specUrl)).thenReturn(specJson);
    Mockito.when(HttpClients.doGetAsString(configUrl)).thenReturn(configJson);
    // Set current CDAP version
    Mockito.doReturn("6.4.0").when(ca).getCurrentVersion();
    // Test plugin auto install
    ca.autoInstallResources("mycapability", Collections.singletonList(new URL("https://my.hub.io")));
    // Verify that the correct version of the plugin was installed
    Set<ArtifactRange> ranges = ImmutableSet.of(ArtifactRanges.parseArtifactRange("system:cdap-data-pipeline[6.3.1,6.4.0]"), ArtifactRanges.parseArtifactRange("system:cdap-data-streams[6.3.1,6.4.0]"));
    Id.Artifact artifact = Id.Artifact.from(Id.Namespace.DEFAULT, "my-plugin", "2.0.0");
    Mockito.verify(artifactRepository, Mockito.times(1)).addArtifact(artifact, mockFile, ranges, ImmutableSet.of());
    Mockito.verify(artifactRepository, Mockito.times(1)).writeArtifactProperties(artifact, properties);
    // Verify that temp file was deleted
    PowerMockito.verifyStatic();
    java.nio.file.Files.deleteIfExists(mockPath);
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) NoOpInternalAuthenticator(io.cdap.cdap.common.internal.remote.NoOpInternalAuthenticator) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) Matchers.anyString(org.mockito.Matchers.anyString) URL(java.net.URL) HubPackage(io.cdap.cdap.internal.capability.autoinstall.HubPackage) Path(java.nio.file.Path) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) Id(io.cdap.cdap.common.id.Id) Spec(io.cdap.cdap.internal.capability.autoinstall.Spec) File(java.io.File) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 2 with HubPackage

use of io.cdap.cdap.internal.capability.autoinstall.HubPackage in project cdap by cdapio.

the class CapabilityApplier method autoInstallResources.

@VisibleForTesting
void autoInstallResources(String capability, @Nullable List<URL> hubs) {
    ExecutorService executor = Executors.newFixedThreadPool(autoInstallThreads, Threads.createDaemonThreadFactory("installer-" + capability + "-%d"));
    try {
        for (URL hub : Optional.ofNullable(hubs).orElse(Collections.emptyList())) {
            HubPackage[] hubPackages;
            try {
                URL url = new URL(hub.getProtocol(), hub.getHost(), hub.getPort(), hub.getPath() + "/packages.json");
                String packagesJson = HttpClients.doGetAsString(url);
                // Deserialize packages.json from hub
                // See https://cdap.atlassian.net/wiki/spaces/DOCS/pages/554401840/Hub+API?src=search#Get-Hub-Catalog
                hubPackages = GSON.fromJson(packagesJson, HubPackage[].class);
            } catch (Exception e) {
                LOG.warn("Failed to get packages.json from {} for capability {}. Ignoring error.", hub, capability, e);
                continue;
            }
            String currentVersion = getCurrentVersion();
            // Get the latest compatible version of each plugin from hub and install it
            List<Future<?>> futures = Arrays.stream(hubPackages).filter(p -> p.versionIsInRange(currentVersion)).collect(Collectors.groupingBy(HubPackage::getName, Collectors.maxBy(Comparator.comparing(HubPackage::getArtifactVersion)))).values().stream().filter(Optional::isPresent).map(Optional::get).map(p -> executor.submit(() -> {
                try {
                    LOG.debug("Installing plugins {} for capability {} from hub {}", p.getName(), capability, hub);
                    p.installPlugin(hub, artifactRepository, tmpDir);
                } catch (Exception e) {
                    LOG.warn("Failed to install plugin {} for capability {} from hub {}. Ignoring error", p.getName(), capability, hub, e);
                }
            })).collect(Collectors.toList());
            for (Future<?> future : futures) {
                try {
                    future.get();
                } catch (InterruptedException | ExecutionException e) {
                    LOG.warn("Ignoring error during plugin install", e);
                }
            }
        }
    } finally {
        executor.shutdownNow();
    }
}
Also used : MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) ApplicationDetail(io.cdap.cdap.proto.ApplicationDetail) Arrays(java.util.Arrays) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) URL(java.net.URL) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) Future(java.util.concurrent.Future) Gson(com.google.gson.Gson) ArtifactSummary(io.cdap.cdap.api.artifact.ArtifactSummary) ArtifactSortOrder(io.cdap.cdap.proto.artifact.ArtifactSortOrder) Map(java.util.Map) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) ApplicationNotFoundException(io.cdap.cdap.common.ApplicationNotFoundException) ProgramTerminator(io.cdap.cdap.internal.app.deploy.ProgramTerminator) Threads(org.apache.twill.common.Threads) EntityResult(io.cdap.cdap.internal.entity.EntityResult) Collection(java.util.Collection) Set(java.util.Set) ApplicationLifecycleService(io.cdap.cdap.internal.app.services.ApplicationLifecycleService) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Objects(java.util.Objects) NamespaceMeta(io.cdap.cdap.proto.NamespaceMeta) List(java.util.List) HubPackage(io.cdap.cdap.internal.capability.autoinstall.HubPackage) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) Optional(java.util.Optional) Constants(io.cdap.cdap.common.conf.Constants) InvalidArtifactException(io.cdap.cdap.common.InvalidArtifactException) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) NotFoundException(io.cdap.cdap.common.NotFoundException) RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) Retries(io.cdap.cdap.common.service.Retries) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) VersionHelper(io.cdap.cdap.gateway.handlers.util.VersionHelper) Arguments(io.cdap.cdap.app.runtime.Arguments) UnauthorizedException(io.cdap.cdap.security.spi.authorization.UnauthorizedException) HashMap(java.util.HashMap) ProgramType(io.cdap.cdap.proto.ProgramType) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ArtifactVersionRange(io.cdap.cdap.api.artifact.ArtifactVersionRange) ArtifactDetail(io.cdap.cdap.internal.app.runtime.artifact.ArtifactDetail) ExecutorService(java.util.concurrent.ExecutorService) Nullable(javax.annotation.Nullable) ProgramLifecycleService(io.cdap.cdap.internal.app.services.ProgramLifecycleService) Logger(org.slf4j.Logger) ProgramId(io.cdap.cdap.proto.id.ProgramId) IOException(java.io.IOException) File(java.io.File) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) NamespaceAdmin(io.cdap.cdap.common.namespace.NamespaceAdmin) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) SystemProgramManagementService(io.cdap.cdap.internal.app.services.SystemProgramManagementService) Collections(java.util.Collections) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) Optional(java.util.Optional) URL(java.net.URL) HubPackage(io.cdap.cdap.internal.capability.autoinstall.HubPackage) ApplicationNotFoundException(io.cdap.cdap.common.ApplicationNotFoundException) InvalidArtifactException(io.cdap.cdap.common.InvalidArtifactException) NotFoundException(io.cdap.cdap.common.NotFoundException) UnauthorizedException(io.cdap.cdap.security.spi.authorization.UnauthorizedException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) ExecutorService(java.util.concurrent.ExecutorService) Future(java.util.concurrent.Future) ExecutionException(java.util.concurrent.ExecutionException) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 3 with HubPackage

use of io.cdap.cdap.internal.capability.autoinstall.HubPackage in project cdap by cdapio.

the class AutoInstallTest method testAutoInstallPlugins.

@Test
public void testAutoInstallPlugins() throws Exception {
    // Setup mocks
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.setInt(Constants.Capability.AUTO_INSTALL_THREADS, 5);
    ArtifactRepository artifactRepository = PowerMockito.mock(ArtifactRepository.class);
    RemoteClientFactory remoteClientFactory = new RemoteClientFactory(null, new NoOpInternalAuthenticator());
    CapabilityApplier capabilityApplier = new CapabilityApplier(null, null, null, null, null, artifactRepository, cConf, remoteClientFactory);
    CapabilityApplier ca = Mockito.spy(capabilityApplier);
    PowerMockito.mockStatic(HttpClients.class);
    PowerMockito.mockStatic(Files.class);
    PowerMockito.mockStatic(File.class);
    PowerMockito.mockStatic(Paths.class);
    PowerMockito.mockStatic(java.nio.file.Files.class);
    File mockFile = TEMP_FOLDER.newFile();
    Mockito.when(File.createTempFile(anyString(), anyString(), any())).thenReturn(mockFile);
    Path mockPath = PowerMockito.mock(Path.class);
    Mockito.when(Paths.get(mockFile.getPath())).thenReturn(mockPath);
    URL packagesUrl = new URL("https://my.hub.io/packages.json");
    HubPackage pkg1 = new HubPackage("my-plugin", "1.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.1.1,6.3.0]", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    HubPackage pkg2 = new HubPackage("my-plugin", "2.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.3.1,6.4.0]", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    HubPackage pkg3 = new HubPackage("my-plugin", "3.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.4.1,7.0.0-SNAPSHOT)", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    String packagesJson = GSON.toJson(ImmutableList.of(pkg1, pkg2, pkg3));
    URL specUrl = new URL("https://my.hub.io/packages/my-plugin/2.0.0/spec.json");
    List<Spec.Action.Argument> arguments = Arrays.asList(new Spec.Action.Argument("config", "my-plugin-2.0.0.json", false), new Spec.Action.Argument("jar", "my-plugin-2.0.0.jar", false), new Spec.Action.Argument("name", "my-plugin", false), new Spec.Action.Argument("version", "2.0.0", false));
    Spec.Action action = new Spec.Action("one_step_deploy_plugin", "Deploy my plugin", arguments);
    Spec spec = new Spec("1.0", "My Plugin", "My Plugin", "Cask", "Cask", 1554766945, "[6.3.1,6.4.0]", Collections.singletonList("hydrator-plugin"), true, Collections.singletonList(action));
    String specJson = GSON.toJson(spec);
    URL configUrl = new URL("https://my.hub.io/packages/my-plugin/2.0.0/my-plugin-2.0.0.json");
    Map<String, String> properties = ImmutableMap.of("key1", "value1", "key2", "value2");
    Map<String, Object> config = ImmutableMap.of("parents", ImmutableList.of("system:cdap-data-pipeline[6.3.1,6.4.0]", "system:cdap-data-streams[6.3.1,6.4.0]"), "properties", properties);
    String configJson = GSON.toJson(config);
    // Mock http requests to hub
    Mockito.when(HttpClients.doGetAsString(packagesUrl)).thenReturn(packagesJson);
    Mockito.when(HttpClients.doGetAsString(specUrl)).thenReturn(specJson);
    Mockito.when(HttpClients.doGetAsString(configUrl)).thenReturn(configJson);
    // Set current CDAP version
    Mockito.doReturn("6.4.0").when(ca).getCurrentVersion();
    // Test plugin auto install
    ca.autoInstallResources("mycapability", Collections.singletonList(new URL("https://my.hub.io")));
    // Verify that the correct version of the plugin was installed
    Set<ArtifactRange> ranges = ImmutableSet.of(ArtifactRanges.parseArtifactRange("system:cdap-data-pipeline[6.3.1,6.4.0]"), ArtifactRanges.parseArtifactRange("system:cdap-data-streams[6.3.1,6.4.0]"));
    Id.Artifact artifact = Id.Artifact.from(Id.Namespace.DEFAULT, "my-plugin", "2.0.0");
    Mockito.verify(artifactRepository, Mockito.times(1)).addArtifact(artifact, mockFile, ranges, ImmutableSet.of());
    Mockito.verify(artifactRepository, Mockito.times(1)).writeArtifactProperties(artifact, properties);
    // Verify that temp file was deleted
    PowerMockito.verifyStatic();
    java.nio.file.Files.deleteIfExists(mockPath);
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) NoOpInternalAuthenticator(io.cdap.cdap.common.internal.remote.NoOpInternalAuthenticator) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) Matchers.anyString(org.mockito.Matchers.anyString) URL(java.net.URL) HubPackage(io.cdap.cdap.internal.capability.autoinstall.HubPackage) Path(java.nio.file.Path) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) Id(io.cdap.cdap.common.id.Id) Spec(io.cdap.cdap.internal.capability.autoinstall.Spec) File(java.io.File) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 4 with HubPackage

use of io.cdap.cdap.internal.capability.autoinstall.HubPackage in project cdap by cdapio.

the class AutoInstallTest method testDeserializePackageJson.

@Test
public void testDeserializePackageJson() throws Exception {
    HubPackage[] deserializedHubPackages = readFile("packages.json", HubPackage[].class);
    HubPackage[] expectedHubPackages = { new HubPackage("plugin-1", "1.0.0", "Description1", "Label1", "Author1", "Org1", "[6.3.0,7.0.0-SNAPSHOT)", 1473901763, false, Arrays.asList("category1"), true, "https://www.foo.com"), new HubPackage("plugin-2", "2.0.0", "Description2", "Label2", "Author2", "Org2", "[6.1.1,6.1.1]", 1510006834, true, Arrays.asList("category2", "category3"), false, null) };
    Assert.assertArrayEquals(expectedHubPackages, deserializedHubPackages);
    Assert.assertTrue(deserializedHubPackages[0].versionIsInRange("6.4.0"));
    Assert.assertFalse(deserializedHubPackages[0].versionIsInRange("7.0.0"));
    Assert.assertTrue(deserializedHubPackages[1].versionIsInRange("6.1.1"));
    Assert.assertFalse(deserializedHubPackages[1].versionIsInRange("6.1.2"));
}
Also used : HubPackage(io.cdap.cdap.internal.capability.autoinstall.HubPackage) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 5 with HubPackage

use of io.cdap.cdap.internal.capability.autoinstall.HubPackage in project cdap by caskdata.

the class AutoInstallTest method testDeserializePackageJson.

@Test
public void testDeserializePackageJson() throws Exception {
    HubPackage[] deserializedHubPackages = readFile("packages.json", HubPackage[].class);
    HubPackage[] expectedHubPackages = { new HubPackage("plugin-1", "1.0.0", "Description1", "Label1", "Author1", "Org1", "[6.3.0,7.0.0-SNAPSHOT)", 1473901763, false, Arrays.asList("category1"), true, "https://www.foo.com"), new HubPackage("plugin-2", "2.0.0", "Description2", "Label2", "Author2", "Org2", "[6.1.1,6.1.1]", 1510006834, true, Arrays.asList("category2", "category3"), false, null) };
    Assert.assertArrayEquals(expectedHubPackages, deserializedHubPackages);
    Assert.assertTrue(deserializedHubPackages[0].versionIsInRange("6.4.0"));
    Assert.assertFalse(deserializedHubPackages[0].versionIsInRange("7.0.0"));
    Assert.assertTrue(deserializedHubPackages[1].versionIsInRange("6.1.1"));
    Assert.assertFalse(deserializedHubPackages[1].versionIsInRange("6.1.2"));
}
Also used : HubPackage(io.cdap.cdap.internal.capability.autoinstall.HubPackage) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

HubPackage (io.cdap.cdap.internal.capability.autoinstall.HubPackage)6 ArtifactRange (io.cdap.cdap.api.artifact.ArtifactRange)4 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)4 RemoteClientFactory (io.cdap.cdap.common.internal.remote.RemoteClientFactory)4 ArtifactRepository (io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository)4 File (java.io.File)3 URL (java.net.URL)3 Test (org.junit.Test)3 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 Gson (com.google.gson.Gson)2 Inject (com.google.inject.Inject)2 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)2 ArtifactScope (io.cdap.cdap.api.artifact.ArtifactScope)2 ArtifactSummary (io.cdap.cdap.api.artifact.ArtifactSummary)2 ArtifactVersionRange (io.cdap.cdap.api.artifact.ArtifactVersionRange)2 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)2 MetadataScope (io.cdap.cdap.api.metadata.MetadataScope)2 Arguments (io.cdap.cdap.app.runtime.Arguments)2 ApplicationNotFoundException (io.cdap.cdap.common.ApplicationNotFoundException)2