Examples with ArtifactRange io.cdap.cdap.api.artifact.ArtifactRange used on opensource projects

Example 36 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by cdapio.

the class ArtifactRepositoryTest method testGreatGrandparentsAreInvalid.

@Test
public void testGreatGrandparentsAreInvalid() throws Exception {
    // create child artifact
    io.cdap.cdap.proto.id.ArtifactId childId = NamespaceId.DEFAULT.artifact("child", "1.0.0");
    Manifest manifest = createManifest(ManifestFields.EXPORT_PACKAGE, Plugin1.class.getPackage().getName());
    File jarFile = createPluginJar(Plugin1.class, new File(tmpDir, "child-1.0.0.jar"), manifest);
    // add the artifact
    Set<ArtifactRange> parents = ImmutableSet.of(new ArtifactRange(APP_ARTIFACT_ID.getNamespace().getId(), APP_ARTIFACT_ID.getName(), new ArtifactVersion("1.0.0"), new ArtifactVersion("2.0.0")));
    artifactRepository.addArtifact(Id.Artifact.fromEntityId(childId), jarFile, parents, null);
    // create grandchild
    io.cdap.cdap.proto.id.ArtifactId grandchildId = NamespaceId.DEFAULT.artifact("grandchild", "1.0.0");
    manifest = createManifest(ManifestFields.EXPORT_PACKAGE, Plugin2.class.getPackage().getName());
    jarFile = createPluginJar(Plugin2.class, new File(tmpDir, "grandchild-1.0.0.jar"), manifest);
    parents = ImmutableSet.of(new ArtifactRange(childId.getNamespace(), childId.getArtifact(), new ArtifactVersion("1.0.0"), new ArtifactVersion("2.0.0")));
    artifactRepository.addArtifact(Id.Artifact.fromEntityId(grandchildId), jarFile, parents, null);
    // try and create great grandchild, should fail
    io.cdap.cdap.proto.id.ArtifactId greatGrandchildId = NamespaceId.DEFAULT.artifact("greatgrandchild", "1.0.0");
    manifest = createManifest(ManifestFields.EXPORT_PACKAGE, Plugin2.class.getPackage().getName());
    jarFile = createPluginJar(Plugin2.class, new File(tmpDir, "greatgrandchild-1.0.0.jar"), manifest);
    parents = ImmutableSet.of(new ArtifactRange(grandchildId.getNamespace(), grandchildId.getArtifact(), new ArtifactVersion("1.0.0"), new ArtifactVersion("2.0.0")));
    try {
        artifactRepository.addArtifact(Id.Artifact.fromEntityId(greatGrandchildId), jarFile, parents, null);
        Assert.fail("Artifact repository is not supposed to allow great grandparents.");
    } catch (InvalidArtifactException e) {
    // expected
    }
}

Example 37 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by cdapio.

the class AutoInstallTest method testAutoInstallPlugins.

@Test
public void testAutoInstallPlugins() throws Exception {
    // Setup mocks
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.setInt(Constants.Capability.AUTO_INSTALL_THREADS, 5);
    ArtifactRepository artifactRepository = PowerMockito.mock(ArtifactRepository.class);
    RemoteClientFactory remoteClientFactory = new RemoteClientFactory(null, new NoOpInternalAuthenticator());
    CapabilityApplier capabilityApplier = new CapabilityApplier(null, null, null, null, null, artifactRepository, cConf, remoteClientFactory);
    CapabilityApplier ca = Mockito.spy(capabilityApplier);
    PowerMockito.mockStatic(HttpClients.class);
    PowerMockito.mockStatic(Files.class);
    PowerMockito.mockStatic(File.class);
    PowerMockito.mockStatic(Paths.class);
    PowerMockito.mockStatic(java.nio.file.Files.class);
    File mockFile = TEMP_FOLDER.newFile();
    Mockito.when(File.createTempFile(anyString(), anyString(), any())).thenReturn(mockFile);
    Path mockPath = PowerMockito.mock(Path.class);
    Mockito.when(Paths.get(mockFile.getPath())).thenReturn(mockPath);
    URL packagesUrl = new URL("https://my.hub.io/packages.json");
    HubPackage pkg1 = new HubPackage("my-plugin", "1.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.1.1,6.3.0]", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    HubPackage pkg2 = new HubPackage("my-plugin", "2.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.3.1,6.4.0]", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    HubPackage pkg3 = new HubPackage("my-plugin", "3.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.4.1,7.0.0-SNAPSHOT)", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    String packagesJson = GSON.toJson(ImmutableList.of(pkg1, pkg2, pkg3));
    URL specUrl = new URL("https://my.hub.io/packages/my-plugin/2.0.0/spec.json");
    List<Spec.Action.Argument> arguments = Arrays.asList(new Spec.Action.Argument("config", "my-plugin-2.0.0.json", false), new Spec.Action.Argument("jar", "my-plugin-2.0.0.jar", false), new Spec.Action.Argument("name", "my-plugin", false), new Spec.Action.Argument("version", "2.0.0", false));
    Spec.Action action = new Spec.Action("one_step_deploy_plugin", "Deploy my plugin", arguments);
    Spec spec = new Spec("1.0", "My Plugin", "My Plugin", "Cask", "Cask", 1554766945, "[6.3.1,6.4.0]", Collections.singletonList("hydrator-plugin"), true, Collections.singletonList(action));
    String specJson = GSON.toJson(spec);
    URL configUrl = new URL("https://my.hub.io/packages/my-plugin/2.0.0/my-plugin-2.0.0.json");
    Map<String, String> properties = ImmutableMap.of("key1", "value1", "key2", "value2");
    Map<String, Object> config = ImmutableMap.of("parents", ImmutableList.of("system:cdap-data-pipeline[6.3.1,6.4.0]", "system:cdap-data-streams[6.3.1,6.4.0]"), "properties", properties);
    String configJson = GSON.toJson(config);
    // Mock http requests to hub
    Mockito.when(HttpClients.doGetAsString(packagesUrl)).thenReturn(packagesJson);
    Mockito.when(HttpClients.doGetAsString(specUrl)).thenReturn(specJson);
    Mockito.when(HttpClients.doGetAsString(configUrl)).thenReturn(configJson);
    // Set current CDAP version
    Mockito.doReturn("6.4.0").when(ca).getCurrentVersion();
    // Test plugin auto install
    ca.autoInstallResources("mycapability", Collections.singletonList(new URL("https://my.hub.io")));
    // Verify that the correct version of the plugin was installed
    Set<ArtifactRange> ranges = ImmutableSet.of(ArtifactRanges.parseArtifactRange("system:cdap-data-pipeline[6.3.1,6.4.0]"), ArtifactRanges.parseArtifactRange("system:cdap-data-streams[6.3.1,6.4.0]"));
    Id.Artifact artifact = Id.Artifact.from(Id.Namespace.DEFAULT, "my-plugin", "2.0.0");
    Mockito.verify(artifactRepository, Mockito.times(1)).addArtifact(artifact, mockFile, ranges, ImmutableSet.of());
    Mockito.verify(artifactRepository, Mockito.times(1)).writeArtifactProperties(artifact, properties);
    // Verify that temp file was deleted
    PowerMockito.verifyStatic();
    java.nio.file.Files.deleteIfExists(mockPath);
}

Example 38 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by cdapio.

the class ArtifactHttpHandlerInternalTest method testRemoteArtifactRepositoryReaderGetArtifactDetails.

/**
 * Test {@link RemoteArtifactRepositoryReader#getArtifactDetails}
 */
@Test
public void testRemoteArtifactRepositoryReaderGetArtifactDetails() throws Exception {
    // Add an artifact with a number of versions
    String systemArtfiactName = "sysApp3";
    final int numVersions = 10;
    List<ArtifactId> artifactIds = new ArrayList<>();
    for (int i = 1; i <= numVersions; i++) {
        String version = String.format("%d.0.0", i);
        ArtifactId systemArtifactId = NamespaceId.SYSTEM.artifact(systemArtfiactName, version);
        addAppAsSystemArtifacts(systemArtifactId);
        artifactIds.add(systemArtifactId);
    }
    ArtifactRepositoryReader remoteReader = getInjector().getInstance(RemoteArtifactRepositoryReader.class);
    ArtifactRange range = null;
    List<ArtifactDetail> details = null;
    ArtifactId systemArtifactId = null;
    ArtifactDetail expectedDetail = null;
    int numLimits = 0;
    range = new ArtifactRange(NamespaceId.SYSTEM.getNamespace(), systemArtfiactName, new ArtifactVersion("1.0.0"), new ArtifactVersion(String.format("%d.0.0", numVersions)));
    // Fetch artifacts with the version in range [1.0.0, numVersions.0.0], but limit == 1 in desc order
    numLimits = 1;
    details = remoteReader.getArtifactDetails(range, numLimits, ArtifactSortOrder.DESC);
    Assert.assertEquals(numLimits, details.size());
    systemArtifactId = NamespaceId.SYSTEM.artifact(systemArtfiactName, String.format("%d.0.0", numVersions));
    expectedDetail = getArtifactDetailFromRepository(Id.Artifact.fromEntityId(systemArtifactId));
    Assert.assertTrue(details.get(numLimits - 1).equals(expectedDetail));
    // Fetch artifacts with the version in range [1.0.0, numVersions.0.0], but limit == 3 in desc order
    numLimits = 3;
    details = remoteReader.getArtifactDetails(range, numLimits, ArtifactSortOrder.DESC);
    Assert.assertEquals(numLimits, details.size());
    for (int i = 0; i < numLimits; i++) {
        systemArtifactId = NamespaceId.SYSTEM.artifact(systemArtfiactName, String.format("%d.0.0", numVersions - i));
        expectedDetail = getArtifactDetailFromRepository(Id.Artifact.fromEntityId(systemArtifactId));
        Assert.assertTrue(details.get(i).equals(expectedDetail));
    }
    // Fetch artifacts with the version in range [1.0.0, numVersions.0.0], but limit == 1 in asec order
    details = remoteReader.getArtifactDetails(range, 1, ArtifactSortOrder.ASC);
    Assert.assertEquals(1, details.size());
    systemArtifactId = NamespaceId.SYSTEM.artifact(systemArtfiactName, "1.0.0");
    expectedDetail = getArtifactDetailFromRepository(Id.Artifact.fromEntityId(systemArtifactId));
    Assert.assertTrue(details.get(0).equals(expectedDetail));
    // Fetch artifacts with the version in range [1.0.0, numVersions.0.0], but limit == 5 in asec order
    numLimits = 5;
    details = remoteReader.getArtifactDetails(range, numLimits, ArtifactSortOrder.ASC);
    Assert.assertEquals(numLimits, details.size());
    for (int i = 0; i < numLimits; i++) {
        systemArtifactId = NamespaceId.SYSTEM.artifact(systemArtfiactName, String.format("%d.0.0", i + 1));
        expectedDetail = getArtifactDetailFromRepository(Id.Artifact.fromEntityId(systemArtifactId));
        Assert.assertTrue(details.get(i).equals(expectedDetail));
    }
    int versionLow = 1;
    int versionHigh = numVersions / 2;
    range = new ArtifactRange(NamespaceId.SYSTEM.getNamespace(), systemArtfiactName, new ArtifactVersion(String.format("%d.0.0", versionLow)), new ArtifactVersion(String.format("%d.0.0", versionHigh)));
    // Fetch artifacts with the version in range [1.0.0, <numVersions/2>.0.0], but limit == numVersions in desc order
    numLimits = numVersions;
    details = remoteReader.getArtifactDetails(range, numLimits, ArtifactSortOrder.DESC);
    Assert.assertEquals(versionHigh - versionLow + 1, details.size());
    for (int i = 0; i < versionHigh - versionLow + 1; i++) {
        systemArtifactId = NamespaceId.SYSTEM.artifact(systemArtfiactName, String.format("%d.0.0", versionHigh - i));
        expectedDetail = getArtifactDetailFromRepository(Id.Artifact.fromEntityId(systemArtifactId));
        Assert.assertTrue(details.get(i).equals(expectedDetail));
    }
}

Example 39 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by cdapio.

the class UnitTestManager method addPluginArtifact.

@Override
public ArtifactManager addPluginArtifact(ArtifactId artifactId, ArtifactId parent, Class<?> pluginClass, Class<?>... pluginClasses) throws Exception {
    Set<ArtifactRange> parents = new HashSet<>();
    parents.add(new ArtifactRange(parent.getParent().getNamespace(), parent.getArtifact(), new ArtifactVersion(parent.getVersion()), true, new ArtifactVersion(parent.getVersion()), true));
    addPluginArtifact(artifactId, parents, pluginClass, pluginClasses);
    return artifactManagerFactory.create(artifactId);
}

Example 40 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by cdapio.

the class ArtifactHttpHandler method addArtifact.

@POST
@Path("/namespaces/{namespace-id}/artifacts/{artifact-name}")
@AuditPolicy(AuditDetail.HEADERS)
public BodyConsumer addArtifact(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") final String namespaceId, @PathParam("artifact-name") final String artifactName, @HeaderParam(VERSION_HEADER) final String artifactVersion, @HeaderParam(EXTENDS_HEADER) final String parentArtifactsStr, @HeaderParam(PLUGINS_HEADER) String pluginClasses) throws NamespaceNotFoundException, BadRequestException {
    final NamespaceId namespace = validateAndGetNamespace(namespaceId);
    // that processes the last http chunk.
    if (artifactVersion != null && !artifactVersion.isEmpty()) {
        ArtifactId artifactId = validateAndGetArtifactId(namespace, artifactName, artifactVersion);
        // If the artifact ID is available, use it to perform an authorization check.
        contextAccessEnforcer.enforce(artifactId, StandardPermission.CREATE);
    } else {
        // If there is no version, we perform an enforceOnParent check in which the entityID is not needed.
        contextAccessEnforcer.enforceOnParent(EntityType.ARTIFACT, namespace, StandardPermission.CREATE);
    }
    final Set<ArtifactRange> parentArtifacts = parseExtendsHeader(namespace, parentArtifactsStr);
    final Set<PluginClass> additionalPluginClasses;
    if (pluginClasses == null || pluginClasses.isEmpty()) {
        additionalPluginClasses = ImmutableSet.of();
    } else {
        try {
            additionalPluginClasses = GSON.fromJson(pluginClasses, PLUGINS_TYPE);
            additionalPluginClasses.forEach(PluginClass::validate);
        } catch (JsonParseException e) {
            throw new BadRequestException(String.format("%s header '%s' is invalid.", PLUGINS_HEADER, pluginClasses), e);
        } catch (IllegalArgumentException e) {
            throw new BadRequestException(String.format("Invalid PluginClasses '%s'.", pluginClasses), e);
        }
    }
    try {
        // copy the artifact contents to local tmp directory
        Files.createDirectories(tmpDir.toPath());
        File destination = File.createTempFile("artifact-", ".jar", tmpDir);
        return new AbstractBodyConsumer(destination) {

            @Override
            protected void onFinish(HttpResponder responder, File uploadedFile) {
                try {
                    String version = (artifactVersion == null || artifactVersion.isEmpty()) ? getBundleVersion(uploadedFile) : artifactVersion;
                    ArtifactId artifactId = validateAndGetArtifactId(namespace, artifactName, version);
                    // add the artifact to the repo
                    artifactRepository.addArtifact(Id.Artifact.fromEntityId(artifactId), uploadedFile, parentArtifacts, additionalPluginClasses);
                    responder.sendString(HttpResponseStatus.OK, "Artifact added successfully");
                } catch (ArtifactRangeNotFoundException e) {
                    responder.sendString(HttpResponseStatus.NOT_FOUND, e.getMessage());
                } catch (ArtifactAlreadyExistsException e) {
                    responder.sendString(HttpResponseStatus.CONFLICT, e.getMessage());
                } catch (WriteConflictException e) {
                    responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Conflict while writing artifact, please try again.");
                } catch (IOException e) {
                    LOG.error("Exception while trying to write artifact {}-{}-{}.", namespaceId, artifactName, artifactVersion, e);
                    responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Error performing IO while writing artifact.");
                } catch (BadRequestException e) {
                    responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
                } catch (UnauthorizedException e) {
                    responder.sendString(HttpResponseStatus.FORBIDDEN, e.getMessage());
                } catch (Exception e) {
                    LOG.error("Error while writing artifact {}-{}-{}", namespaceId, artifactName, artifactVersion, e);
                    responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Error while adding artifact.");
                }
            }

            private String getBundleVersion(File file) throws BadRequestException, IOException {
                try (JarFile jarFile = new JarFile(file)) {
                    Manifest manifest = jarFile.getManifest();
                    if (manifest == null) {
                        throw new BadRequestException("Unable to derive version from artifact because it does not contain a manifest. " + "Please package the jar with a manifest, or explicitly specify the artifact version.");
                    }
                    Attributes attributes = manifest.getMainAttributes();
                    String version = attributes == null ? null : attributes.getValue(ManifestFields.BUNDLE_VERSION);
                    if (version == null) {
                        throw new BadRequestException("Unable to derive version from artifact because manifest does not contain Bundle-Version attribute. " + "Please include Bundle-Version in the manifest, or explicitly specify the artifact version.");
                    }
                    return version;
                } catch (ZipException e) {
                    throw new BadRequestException("Artifact is not in zip format. Please make sure it is a jar file.");
                }
            }
        };
    } catch (IOException e) {
        LOG.error("Exception creating temp file to place artifact {} contents", artifactName, e);
        responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Server error creating temp file for artifact.");
        return null;
    }
}