Search in sources :

Example 1 with PluginClass

use of io.cdap.cdap.api.plugin.PluginClass in project cdap by caskdata.

the class ArtifactStore method addPluginsInRangeToMap.

private void addPluginsInRangeToMap(final NamespaceId namespace, List<Id.Artifact> parentArtifacts, Iterator<StructuredRow> iterator, SortedMap<ArtifactDescriptor, PluginClass> plugins, @Nullable Predicate<io.cdap.cdap.proto.id.ArtifactId> range, int limit) {
    // if predicate is null,
    // filter out plugins whose artifacts are not in the system namespace and not in this namespace
    range = range != null ? range : input -> NamespaceId.SYSTEM.equals(input.getParent()) || input.getParent().equals(namespace);
    while (iterator.hasNext()) {
        StructuredRow row = iterator.next();
        ImmutablePair<ArtifactDescriptor, PluginData> pluginPair = getPlugin(row, range);
        if (pluginPair == null) {
            continue;
        }
        PluginData pluginData = pluginPair.getSecond();
        // filter out plugins that don't extend this version of the parent artifact
        for (Id.Artifact parentArtifactId : parentArtifacts) {
            if (pluginData.isUsableBy(parentArtifactId.toEntityId()) && isAllowed(pluginData.pluginClass)) {
                plugins.put(pluginPair.getFirst(), pluginData.pluginClass);
                break;
            }
        }
        if (limit < plugins.size()) {
            plugins.remove(plugins.lastKey());
        }
    }
}
Also used : Arrays(java.util.Arrays) ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) TransactionRunners(io.cdap.cdap.spi.data.transaction.TransactionRunners) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Spliterators(java.util.Spliterators) Inject(com.google.inject.Inject) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) ArtifactClasses(io.cdap.cdap.api.artifact.ArtifactClasses) Fields(io.cdap.cdap.spi.data.table.field.Fields) GsonBuilder(com.google.gson.GsonBuilder) Gson(com.google.gson.Gson) ArtifactSortOrder(io.cdap.cdap.proto.artifact.ArtifactSortOrder) Map(java.util.Map) Field(io.cdap.cdap.spi.data.table.field.Field) URI(java.net.URI) Collector(java.util.stream.Collector) StoreDefinition(io.cdap.cdap.store.StoreDefinition) Predicate(java.util.function.Predicate) Collection(java.util.Collection) Set(java.util.Set) ApplicationClass(io.cdap.cdap.api.artifact.ApplicationClass) StructuredTableContext(io.cdap.cdap.spi.data.StructuredTableContext) Collectors(java.util.stream.Collectors) PluginClass(io.cdap.cdap.api.plugin.PluginClass) BinaryOperator(java.util.function.BinaryOperator) Id(io.cdap.cdap.common.id.Id) List(java.util.List) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) Optional(java.util.Optional) Constants(io.cdap.cdap.common.conf.Constants) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) ArtifactAlreadyExistsException(io.cdap.cdap.common.ArtifactAlreadyExistsException) SortedMap(java.util.SortedMap) SchemaTypeAdapter(io.cdap.cdap.internal.io.SchemaTypeAdapter) TransactionException(io.cdap.cdap.spi.data.transaction.TransactionException) Location(org.apache.twill.filesystem.Location) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) PluginNotExistsException(io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException) StructuredTableId(io.cdap.cdap.spi.data.table.StructuredTableId) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) Locations(io.cdap.cdap.common.io.Locations) StreamSupport(java.util.stream.StreamSupport) LinkedHashSet(java.util.LinkedHashSet) Nullable(javax.annotation.Nullable) OutputStream(java.io.OutputStream) Iterator(java.util.Iterator) Files(java.nio.file.Files) Throwables(com.google.common.base.Throwables) Impersonator(io.cdap.cdap.security.impersonation.Impersonator) IOException(java.io.IOException) LocationFactory(org.apache.twill.filesystem.LocationFactory) Maps(com.google.common.collect.Maps) Schema(io.cdap.cdap.api.data.schema.Schema) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) TableNotFoundException(io.cdap.cdap.spi.data.TableNotFoundException) File(java.io.File) MinMaxPriorityQueue(com.google.common.collect.MinMaxPriorityQueue) EntityImpersonator(io.cdap.cdap.security.impersonation.EntityImpersonator) AbstractMap(java.util.AbstractMap) Requirements(io.cdap.cdap.api.plugin.Requirements) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) TreeMap(java.util.TreeMap) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) StructuredTable(io.cdap.cdap.spi.data.StructuredTable) Range(io.cdap.cdap.spi.data.table.field.Range) NamespacePathLocator(io.cdap.cdap.common.namespace.NamespacePathLocator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Collections(java.util.Collections) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) StructuredTableId(io.cdap.cdap.spi.data.table.StructuredTableId) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId)

Example 2 with PluginClass

use of io.cdap.cdap.api.plugin.PluginClass in project cdap by caskdata.

the class ArtifactStore method filterPlugins.

/**
 * Filters the plugins contained in the {@link ArtifactMeta} by using {@link #isAllowed(PluginClass)}
 *
 * @param artifactMeta the artifact meta
 * @return the {@link ArtifactMeta} containing only the plugins which are not excluded
 */
private ArtifactMeta filterPlugins(ArtifactMeta artifactMeta) {
    ArtifactClasses classes = artifactMeta.getClasses();
    Set<PluginClass> filteredPlugins = classes.getPlugins().stream().filter(this::isAllowed).collect(Collectors.toSet());
    ArtifactClasses filteredClasses = ArtifactClasses.builder().addApps(classes.getApps()).addPlugins(filteredPlugins).build();
    return new ArtifactMeta(filteredClasses, artifactMeta.getUsableBy(), artifactMeta.getProperties());
}
Also used : ArtifactClasses(io.cdap.cdap.api.artifact.ArtifactClasses) PluginClass(io.cdap.cdap.api.plugin.PluginClass)

Example 3 with PluginClass

use of io.cdap.cdap.api.plugin.PluginClass in project cdap by caskdata.

the class ArtifactStore method getPluginsInArtifact.

private SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginsInArtifact(StructuredTable artifactDataTable, Id.Artifact artifactId, Predicate<PluginClass> filter) throws ArtifactNotFoundException, IOException {
    SortedMap<ArtifactDescriptor, Set<PluginClass>> result = new TreeMap<>();
    // Make sure the artifact exists
    ArtifactCell artifactCell = new ArtifactCell(artifactId);
    Optional<StructuredRow> row = artifactDataTable.read(artifactCell.keys);
    if (!row.isPresent()) {
        throw new ArtifactNotFoundException(artifactId.toEntityId());
    }
    // include any plugin classes that are inside the artifact itself and is accepted by the filter
    ArtifactData artifactData = GSON.fromJson(row.get().getString(StoreDefinition.ArtifactStore.ARTIFACT_DATA_FIELD), ArtifactData.class);
    Set<PluginClass> plugins = artifactData.meta.getClasses().getPlugins().stream().filter(filter).collect(Collectors.toCollection(LinkedHashSet::new));
    if (!plugins.isEmpty()) {
        Location location = Locations.getLocationFromAbsolutePath(locationFactory, artifactData.getLocationPath());
        ArtifactDescriptor descriptor = new ArtifactDescriptor(artifactId.getNamespace().getId(), artifactId.toArtifactId(), location);
        result.put(descriptor, plugins);
    }
    return result;
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) TreeMap(java.util.TreeMap) PluginClass(io.cdap.cdap.api.plugin.PluginClass) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) Location(org.apache.twill.filesystem.Location)

Example 4 with PluginClass

use of io.cdap.cdap.api.plugin.PluginClass in project cdap by caskdata.

the class ArtifactStore method getPluginClasses.

/**
 * Get all plugin classes of the given type and name that extend the given parent artifact.
 * Results are returned as a map from plugin artifact to plugins in that artifact.
 *
 * @param namespace the namespace to search for plugins. The system namespace is always included.
 * @param parentArtifactRange the parent artifact range to find plugins for
 * @param type the type of plugin to look for
 * @param name the name of the plugin to look for
 * @param pluginRange the predicate for the plugins
 * @param limit the limit number of the result
 * @param order the order of the result
 * @return an unmodifiable map of plugin artifact to plugin classes of the given type and name, accessible by the
 *         given artifact. The map will never be null, and will never be empty.
 * @throws PluginNotExistsException if no plugin with the given type and name exists in the namespace
 * @throws IOException if there was an exception reading metadata from the metastore
 */
public SortedMap<ArtifactDescriptor, PluginClass> getPluginClasses(NamespaceId namespace, ArtifactRange parentArtifactRange, String type, String name, @Nullable final Predicate<io.cdap.cdap.proto.id.ArtifactId> pluginRange, int limit, ArtifactSortOrder order) throws IOException, ArtifactNotFoundException, PluginNotExistsException {
    SortedMap<ArtifactDescriptor, PluginClass> result = TransactionRunners.run(transactionRunner, context -> {
        StructuredTable artifactDataTable = getTable(context, StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);
        List<ArtifactDetail> parentArtifactDetails = getArtifacts(artifactDataTable, parentArtifactRange, Integer.MAX_VALUE, null);
        if (parentArtifactDetails.isEmpty()) {
            throw new ArtifactNotFoundException(parentArtifactRange.getNamespace(), parentArtifactRange.getName());
        }
        SortedMap<ArtifactDescriptor, PluginClass> plugins = order == ArtifactSortOrder.DESC ? new TreeMap<>(Collections.reverseOrder()) : new TreeMap<>();
        List<Id.Artifact> parentArtifacts = new ArrayList<>();
        for (ArtifactDetail parentArtifactDetail : parentArtifactDetails) {
            parentArtifacts.add(Id.Artifact.from(Id.Namespace.from(parentArtifactRange.getNamespace()), parentArtifactDetail.getDescriptor().getArtifactId()));
            Set<PluginClass> parentPlugins = parentArtifactDetail.getMeta().getClasses().getPlugins();
            for (PluginClass pluginClass : parentPlugins) {
                if (pluginClass.getName().equals(name) && pluginClass.getType().equals(type) && isAllowed(pluginClass)) {
                    plugins.put(parentArtifactDetail.getDescriptor(), pluginClass);
                    break;
                }
            }
        }
        // Add all plugins that extends from the given set of parents
        StructuredTable pluginTable = getTable(context, StoreDefinition.ArtifactStore.PLUGIN_DATA_TABLE);
        PluginKeyPrefix pluginKey = new PluginKeyPrefix(parentArtifactRange.getNamespace(), parentArtifactRange.getName(), type, name);
        try (CloseableIterator<StructuredRow> iterator = pluginTable.scan(Range.singleton(pluginKey.keys), Integer.MAX_VALUE)) {
            addPluginsInRangeToMap(namespace, parentArtifacts, iterator, plugins, pluginRange, limit);
        }
        // Add all universal plugins
        StructuredTable uniPluginTable = getTable(context, StoreDefinition.ArtifactStore.UNIV_PLUGIN_DATA_TABLE);
        for (String ns : Arrays.asList(namespace.getNamespace(), NamespaceId.SYSTEM.getNamespace())) {
            UniversalPluginKeyPrefix universalPluginKey = new UniversalPluginKeyPrefix(ns, type, name);
            try (CloseableIterator<StructuredRow> iterator = uniPluginTable.scan(Range.singleton(universalPluginKey.keys), Integer.MAX_VALUE)) {
                addPluginsInRangeToMap(namespace, parentArtifacts, iterator, plugins, pluginRange, limit);
            }
        }
        return Collections.unmodifiableSortedMap(plugins);
    }, IOException.class, ArtifactNotFoundException.class);
    if (result.isEmpty()) {
        throw new PluginNotExistsException(new NamespaceId(parentArtifactRange.getNamespace()), type, name);
    }
    return result;
}
Also used : StructuredTable(io.cdap.cdap.spi.data.StructuredTable) ArrayList(java.util.ArrayList) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) PluginNotExistsException(io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) PluginClass(io.cdap.cdap.api.plugin.PluginClass) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException)

Example 5 with PluginClass

use of io.cdap.cdap.api.plugin.PluginClass in project cdap by caskdata.

the class ArtifactStore method deleteMeta.

private void deleteMeta(StructuredTableContext context, Id.Artifact artifactId, ArtifactData oldMeta) throws IOException {
    // delete old artifact data
    StructuredTable artifactTable = getTable(context, StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);
    ArtifactCell artifactCell = new ArtifactCell(artifactId);
    artifactTable.delete(artifactCell.keys);
    // delete old appclass metadata
    StructuredTable appClassTable = getTable(context, StoreDefinition.ArtifactStore.APP_DATA_TABLE);
    for (ApplicationClass appClass : oldMeta.meta.getClasses().getApps()) {
        AppClassKey appClassKey = new AppClassKey(artifactId.getNamespace().toEntityId(), appClass.getClassName());
        deleteRangeFromTable(appClassTable, Range.singleton(appClassKey.keys));
    }
    // delete old plugins, we loop twice to only access to one table at a time to prevent deadlock
    StructuredTable pluginDataTable = getTable(context, StoreDefinition.ArtifactStore.PLUGIN_DATA_TABLE);
    for (PluginClass pluginClass : oldMeta.meta.getClasses().getPlugins()) {
        // delete metadata for each artifact this plugin extends
        for (ArtifactRange artifactRange : oldMeta.meta.getUsableBy()) {
            // these four fields are prefixes of the plugin table primary keys
            PluginKeyPrefix pluginKey = new PluginKeyPrefix(artifactRange.getNamespace(), artifactRange.getName(), pluginClass.getType(), pluginClass.getName());
            pluginDataTable.delete(concatFields(pluginKey.keys, artifactCell.keys));
        }
    }
    // Delete the universal plugin row
    StructuredTable uniPluginTable = getTable(context, StoreDefinition.ArtifactStore.UNIV_PLUGIN_DATA_TABLE);
    for (PluginClass pluginClass : oldMeta.meta.getClasses().getPlugins()) {
        if (oldMeta.meta.getUsableBy().isEmpty()) {
            UniversalPluginKeyPrefix pluginKey = new UniversalPluginKeyPrefix(artifactId.getNamespace().getId(), pluginClass.getType(), pluginClass.getName());
            uniPluginTable.delete(concatFields(pluginKey.keys, artifactCell.keys));
        }
    }
    // delete the old jar file
    try {
        new EntityImpersonator(artifactId.toEntityId(), impersonator).impersonate(() -> {
            Locations.getLocationFromAbsolutePath(locationFactory, oldMeta.getLocationPath()).delete();
            return null;
        });
    } catch (IOException ioe) {
        throw ioe;
    } catch (Exception e) {
        // this should not happen
        throw Throwables.propagate(e);
    }
}
Also used : StructuredTable(io.cdap.cdap.spi.data.StructuredTable) EntityImpersonator(io.cdap.cdap.security.impersonation.EntityImpersonator) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) ApplicationClass(io.cdap.cdap.api.artifact.ApplicationClass) IOException(java.io.IOException) PluginClass(io.cdap.cdap.api.plugin.PluginClass) ArtifactAlreadyExistsException(io.cdap.cdap.common.ArtifactAlreadyExistsException) TransactionException(io.cdap.cdap.spi.data.transaction.TransactionException) PluginNotExistsException(io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException) IOException(java.io.IOException) TableNotFoundException(io.cdap.cdap.spi.data.TableNotFoundException) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException)

Aggregations

PluginClass (io.cdap.cdap.api.plugin.PluginClass)53 Test (org.junit.Test)27 ArtifactId (io.cdap.cdap.proto.id.ArtifactId)23 ArtifactRange (io.cdap.cdap.api.artifact.ArtifactRange)22 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)22 ArtifactVersion (io.cdap.cdap.api.artifact.ArtifactVersion)20 Id (io.cdap.cdap.common.id.Id)19 Set (java.util.Set)17 ImmutableSet (com.google.common.collect.ImmutableSet)14 File (java.io.File)14 Map (java.util.Map)14 PluginNotExistsException (io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException)13 PluginPropertyField (io.cdap.cdap.api.plugin.PluginPropertyField)12 SortedMap (java.util.SortedMap)12 HashMap (java.util.HashMap)10 Location (org.apache.twill.filesystem.Location)10 ApplicationClass (io.cdap.cdap.api.artifact.ApplicationClass)9 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)9 ArtifactNotFoundException (io.cdap.cdap.common.ArtifactNotFoundException)9 IOException (java.io.IOException)9