Examples with ArtifactRange io.cdap.cdap.api.artifact.ArtifactRange used on opensource projects

Example 6 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by caskdata.

the class ArtifactHttpHandler method addArtifact.

@POST
@Path("/namespaces/{namespace-id}/artifacts/{artifact-name}")
@AuditPolicy(AuditDetail.HEADERS)
public BodyConsumer addArtifact(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") final String namespaceId, @PathParam("artifact-name") final String artifactName, @HeaderParam(VERSION_HEADER) final String artifactVersion, @HeaderParam(EXTENDS_HEADER) final String parentArtifactsStr, @HeaderParam(PLUGINS_HEADER) String pluginClasses) throws NamespaceNotFoundException, BadRequestException {
    final NamespaceId namespace = validateAndGetNamespace(namespaceId);
    // that processes the last http chunk.
    if (artifactVersion != null && !artifactVersion.isEmpty()) {
        ArtifactId artifactId = validateAndGetArtifactId(namespace, artifactName, artifactVersion);
        // If the artifact ID is available, use it to perform an authorization check.
        contextAccessEnforcer.enforce(artifactId, StandardPermission.CREATE);
    } else {
        // If there is no version, we perform an enforceOnParent check in which the entityID is not needed.
        contextAccessEnforcer.enforceOnParent(EntityType.ARTIFACT, namespace, StandardPermission.CREATE);
    }
    final Set<ArtifactRange> parentArtifacts = parseExtendsHeader(namespace, parentArtifactsStr);
    final Set<PluginClass> additionalPluginClasses;
    if (pluginClasses == null || pluginClasses.isEmpty()) {
        additionalPluginClasses = ImmutableSet.of();
    } else {
        try {
            additionalPluginClasses = GSON.fromJson(pluginClasses, PLUGINS_TYPE);
            additionalPluginClasses.forEach(PluginClass::validate);
        } catch (JsonParseException e) {
            throw new BadRequestException(String.format("%s header '%s' is invalid.", PLUGINS_HEADER, pluginClasses), e);
        } catch (IllegalArgumentException e) {
            throw new BadRequestException(String.format("Invalid PluginClasses '%s'.", pluginClasses), e);
        }
    }
    try {
        // copy the artifact contents to local tmp directory
        Files.createDirectories(tmpDir.toPath());
        File destination = File.createTempFile("artifact-", ".jar", tmpDir);
        return new AbstractBodyConsumer(destination) {

            @Override
            protected void onFinish(HttpResponder responder, File uploadedFile) {
                try {
                    String version = (artifactVersion == null || artifactVersion.isEmpty()) ? getBundleVersion(uploadedFile) : artifactVersion;
                    ArtifactId artifactId = validateAndGetArtifactId(namespace, artifactName, version);
                    // add the artifact to the repo
                    artifactRepository.addArtifact(Id.Artifact.fromEntityId(artifactId), uploadedFile, parentArtifacts, additionalPluginClasses);
                    responder.sendString(HttpResponseStatus.OK, "Artifact added successfully");
                } catch (ArtifactRangeNotFoundException e) {
                    responder.sendString(HttpResponseStatus.NOT_FOUND, e.getMessage());
                } catch (ArtifactAlreadyExistsException e) {
                    responder.sendString(HttpResponseStatus.CONFLICT, e.getMessage());
                } catch (WriteConflictException e) {
                    responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Conflict while writing artifact, please try again.");
                } catch (IOException e) {
                    LOG.error("Exception while trying to write artifact {}-{}-{}.", namespaceId, artifactName, artifactVersion, e);
                    responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Error performing IO while writing artifact.");
                } catch (BadRequestException e) {
                    responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
                } catch (UnauthorizedException e) {
                    responder.sendString(HttpResponseStatus.FORBIDDEN, e.getMessage());
                } catch (Exception e) {
                    LOG.error("Error while writing artifact {}-{}-{}", namespaceId, artifactName, artifactVersion, e);
                    responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Error while adding artifact.");
                }
            }

            private String getBundleVersion(File file) throws BadRequestException, IOException {
                try (JarFile jarFile = new JarFile(file)) {
                    Manifest manifest = jarFile.getManifest();
                    if (manifest == null) {
                        throw new BadRequestException("Unable to derive version from artifact because it does not contain a manifest. " + "Please package the jar with a manifest, or explicitly specify the artifact version.");
                    }
                    Attributes attributes = manifest.getMainAttributes();
                    String version = attributes == null ? null : attributes.getValue(ManifestFields.BUNDLE_VERSION);
                    if (version == null) {
                        throw new BadRequestException("Unable to derive version from artifact because manifest does not contain Bundle-Version attribute. " + "Please include Bundle-Version in the manifest, or explicitly specify the artifact version.");
                    }
                    return version;
                } catch (ZipException e) {
                    throw new BadRequestException("Artifact is not in zip format. Please make sure it is a jar file.");
                }
            }
        };
    } catch (IOException e) {
        LOG.error("Exception creating temp file to place artifact {} contents", artifactName, e);
        responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Server error creating temp file for artifact.");
        return null;
    }
}

Example 7 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by caskdata.

the class ArtifactHttpHandler method parseExtendsHeader.

// find out if this artifact extends other artifacts. If so, there will be a header like
// 'Artifact-Extends: <name>[<lowerversion>,<upperversion>]/<name>[<lowerversion>,<upperversion>]:
// for example: 'Artifact-Extends: etl-batch[1.0.0,2.0.0]/etl-realtime[1.0.0:3.0.0]
private Set<ArtifactRange> parseExtendsHeader(NamespaceId namespace, String extendsHeader) throws BadRequestException {
    Set<ArtifactRange> parentArtifacts = Sets.newHashSet();
    if (extendsHeader != null && !extendsHeader.isEmpty()) {
        for (String parent : Splitter.on('/').split(extendsHeader)) {
            parent = parent.trim();
            ArtifactRange range;
            // try parsing it as a namespaced range like system:etl-batch[1.0.0,2.0.0)
            try {
                range = ArtifactRanges.parseArtifactRange(parent);
                // only support extending an artifact that is in the same namespace, or system namespace
                if (!NamespaceId.SYSTEM.getNamespace().equals(range.getNamespace()) && !namespace.getNamespace().equals(range.getNamespace())) {
                    throw new BadRequestException(String.format("Parent artifact %s must be in the same namespace or a system artifact.", parent));
                }
            } catch (InvalidArtifactRangeException e) {
                // if this failed, try parsing as a non-namespaced range like etl-batch[1.0.0,2.0.0)
                try {
                    range = ArtifactRanges.parseArtifactRange(namespace.getNamespace(), parent);
                } catch (InvalidArtifactRangeException e1) {
                    throw new BadRequestException(String.format("Invalid artifact range %s: %s", parent, e1.getMessage()));
                }
            }
            parentArtifacts.add(range);
        }
    }
    return parentArtifacts;
}

Example 8 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by caskdata.

the class CapabilityApplier method shouldDeployApp.

// Returns true if capability applier should try to deploy this application. 2 conditions when it returns true:
// 1. Either the application is not deployed before.
// 2. If application is deployed before then the app artifact of the deployed application is not the latest one
// available.
private boolean shouldDeployApp(ApplicationId applicationId, SystemApplication application) throws Exception {
    ApplicationDetail currAppDetail;
    try {
        currAppDetail = applicationLifecycleService.getAppDetail(applicationId);
    } catch (ApplicationNotFoundException exception) {
        return true;
    }
    // Compare if the app artifact version of currently deployed application with highest version of app artifact
    // available. If it's not same, capability applier should redeploy application.
    ArtifactSummary summary = application.getArtifact();
    NamespaceId artifactNamespace = ArtifactScope.SYSTEM.equals(summary.getScope()) ? NamespaceId.SYSTEM : applicationId.getParent();
    ArtifactRange range = new ArtifactRange(artifactNamespace.getNamespace(), summary.getName(), ArtifactVersionRange.parse(summary.getVersion()));
    // this method will not throw ArtifactNotFoundException, if no artifacts in the range, we are expecting an empty
    // collection returned.
    List<ArtifactDetail> artifactDetail = artifactRepository.getArtifactDetails(range, 1, ArtifactSortOrder.DESC);
    if (artifactDetail.isEmpty()) {
        throw new ArtifactNotFoundException(range.getNamespace(), range.getName());
    }
    ArtifactId latestArtifactId = artifactDetail.get(0).getDescriptor().getArtifactId();
    // same artifact. If same means no need to deploy the application again.
    return !currAppDetail.getArtifact().getVersion().equals(latestArtifactId.getVersion().getVersion());
}

Example 9 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by caskdata.

the class AutoInstallTest method testAutoInstallPlugins.

@Test
public void testAutoInstallPlugins() throws Exception {
    // Setup mocks
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.setInt(Constants.Capability.AUTO_INSTALL_THREADS, 5);
    ArtifactRepository artifactRepository = PowerMockito.mock(ArtifactRepository.class);
    RemoteClientFactory remoteClientFactory = new RemoteClientFactory(null, new NoOpInternalAuthenticator());
    CapabilityApplier capabilityApplier = new CapabilityApplier(null, null, null, null, null, artifactRepository, cConf, remoteClientFactory);
    CapabilityApplier ca = Mockito.spy(capabilityApplier);
    PowerMockito.mockStatic(HttpClients.class);
    PowerMockito.mockStatic(Files.class);
    PowerMockito.mockStatic(File.class);
    PowerMockito.mockStatic(Paths.class);
    PowerMockito.mockStatic(java.nio.file.Files.class);
    File mockFile = TEMP_FOLDER.newFile();
    Mockito.when(File.createTempFile(anyString(), anyString(), any())).thenReturn(mockFile);
    Path mockPath = PowerMockito.mock(Path.class);
    Mockito.when(Paths.get(mockFile.getPath())).thenReturn(mockPath);
    URL packagesUrl = new URL("https://my.hub.io/packages.json");
    HubPackage pkg1 = new HubPackage("my-plugin", "1.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.1.1,6.3.0]", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    HubPackage pkg2 = new HubPackage("my-plugin", "2.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.3.1,6.4.0]", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    HubPackage pkg3 = new HubPackage("my-plugin", "3.0.0", "My Plugin", "My Plugin", "Cask", "Cask", "[6.4.1,7.0.0-SNAPSHOT)", 1554766945, true, Collections.singletonList("hydrator-plugin"), false, null);
    String packagesJson = GSON.toJson(ImmutableList.of(pkg1, pkg2, pkg3));
    URL specUrl = new URL("https://my.hub.io/packages/my-plugin/2.0.0/spec.json");
    List<Spec.Action.Argument> arguments = Arrays.asList(new Spec.Action.Argument("config", "my-plugin-2.0.0.json", false), new Spec.Action.Argument("jar", "my-plugin-2.0.0.jar", false), new Spec.Action.Argument("name", "my-plugin", false), new Spec.Action.Argument("version", "2.0.0", false));
    Spec.Action action = new Spec.Action("one_step_deploy_plugin", "Deploy my plugin", arguments);
    Spec spec = new Spec("1.0", "My Plugin", "My Plugin", "Cask", "Cask", 1554766945, "[6.3.1,6.4.0]", Collections.singletonList("hydrator-plugin"), true, Collections.singletonList(action));
    String specJson = GSON.toJson(spec);
    URL configUrl = new URL("https://my.hub.io/packages/my-plugin/2.0.0/my-plugin-2.0.0.json");
    Map<String, String> properties = ImmutableMap.of("key1", "value1", "key2", "value2");
    Map<String, Object> config = ImmutableMap.of("parents", ImmutableList.of("system:cdap-data-pipeline[6.3.1,6.4.0]", "system:cdap-data-streams[6.3.1,6.4.0]"), "properties", properties);
    String configJson = GSON.toJson(config);
    // Mock http requests to hub
    Mockito.when(HttpClients.doGetAsString(packagesUrl)).thenReturn(packagesJson);
    Mockito.when(HttpClients.doGetAsString(specUrl)).thenReturn(specJson);
    Mockito.when(HttpClients.doGetAsString(configUrl)).thenReturn(configJson);
    // Set current CDAP version
    Mockito.doReturn("6.4.0").when(ca).getCurrentVersion();
    // Test plugin auto install
    ca.autoInstallResources("mycapability", Collections.singletonList(new URL("https://my.hub.io")));
    // Verify that the correct version of the plugin was installed
    Set<ArtifactRange> ranges = ImmutableSet.of(ArtifactRanges.parseArtifactRange("system:cdap-data-pipeline[6.3.1,6.4.0]"), ArtifactRanges.parseArtifactRange("system:cdap-data-streams[6.3.1,6.4.0]"));
    Id.Artifact artifact = Id.Artifact.from(Id.Namespace.DEFAULT, "my-plugin", "2.0.0");
    Mockito.verify(artifactRepository, Mockito.times(1)).addArtifact(artifact, mockFile, ranges, ImmutableSet.of());
    Mockito.verify(artifactRepository, Mockito.times(1)).writeArtifactProperties(artifact, properties);
    // Verify that temp file was deleted
    PowerMockito.verifyStatic();
    java.nio.file.Files.deleteIfExists(mockPath);
}

Example 10 with ArtifactRange

use of io.cdap.cdap.api.artifact.ArtifactRange in project cdap by caskdata.

the class ApplicationLifecycleService method deployApp.

/**
 * Deploy an application using the specified artifact and configuration. When an app is deployed, the Application
 * class is instantiated and configure() is called in order to generate an {@link ApplicationSpecification}.
 * Programs, datasets, and streams are created based on the specification before the spec is persisted in the
 * {@link Store}. This method can create a new application as well as update an existing one.
 *
 * @param namespace the namespace to deploy the app to
 * @param appName the name of the app. If null, the name will be set based on the application spec
 * @param summary the artifact summary of the app
 * @param configStr the configuration to send to the application when generating the application specification
 * @param programTerminator a program terminator that will stop programs that are removed when updating an app.
 *                          For example, if an update removes a flow, the terminator defines how to stop that flow.
 * @param ownerPrincipal the kerberos principal of the application owner
 * @param updateSchedules specifies if schedules of the workflow have to be updated,
 *                        if null value specified by the property "app.deploy.update.schedules" will be used.
 * @param isPreview whether the app deployment is for preview
 * @param userProps the user properties for the app deployment, this is basically used for preview deployment
 * @return information about the deployed application
 * @throws InvalidArtifactException if the artifact does not contain any application classes
 * @throws IOException if there was an IO error reading artifact detail from the meta store
 * @throws ArtifactNotFoundException if the specified artifact does not exist
 * @throws Exception if there was an exception during the deployment pipeline. This exception will often wrap
 *                   the actual exception
 */
public ApplicationWithPrograms deployApp(NamespaceId namespace, @Nullable String appName, @Nullable String appVersion, ArtifactSummary summary, @Nullable String configStr, ProgramTerminator programTerminator, @Nullable KerberosPrincipalId ownerPrincipal, @Nullable Boolean updateSchedules, boolean isPreview, Map<String, String> userProps) throws Exception {
    NamespaceId artifactNamespace = ArtifactScope.SYSTEM.equals(summary.getScope()) ? NamespaceId.SYSTEM : namespace;
    ArtifactRange range = new ArtifactRange(artifactNamespace.getNamespace(), summary.getName(), ArtifactVersionRange.parse(summary.getVersion()));
    // this method will not throw ArtifactNotFoundException, if no artifacts in the range, we are expecting an empty
    // collection returned.
    List<ArtifactDetail> artifactDetail = artifactRepository.getArtifactDetails(range, 1, ArtifactSortOrder.DESC);
    if (artifactDetail.isEmpty()) {
        throw new ArtifactNotFoundException(range.getNamespace(), range.getName());
    }
    return deployApp(namespace, appName, appVersion, configStr, programTerminator, artifactDetail.iterator().next(), ownerPrincipal, updateSchedules == null ? appUpdateSchedules : updateSchedules, isPreview, userProps);
}