Search in sources :

Example 21 with Range

use of io.cdap.cdap.spi.data.table.field.Range in project cdap by cdapio.

the class ArtifactStore method getArtifacts.

/**
 * Get information about all artifacts in the given namespace. If there are no artifacts in the namespace,
 * this will return an empty list. Note that existence of the namespace is not checked.
 *
 * @param namespace the namespace to get artifact information about
 * @return unmodifiable list of artifact info about every artifact in the given namespace
 * @throws IOException if there was an exception reading the artifact information from the metastore
 */
public List<ArtifactDetail> getArtifacts(final NamespaceId namespace) throws IOException {
    return TransactionRunners.run(transactionRunner, context -> {
        StructuredTable table = getTable(context, StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);
        List<ArtifactDetail> artifacts = Lists.newArrayList();
        Range scanRange = createArtifactScanRange(namespace);
        // TODO: CDAP-14636 add scan method without limit
        try (CloseableIterator<StructuredRow> iterator = table.scan(scanRange, Integer.MAX_VALUE)) {
            getArtifacts(iterator, Integer.MAX_VALUE, null, () -> artifacts);
        }
        return Collections.unmodifiableList(artifacts);
    }, IOException.class);
}
Also used : StructuredTable(io.cdap.cdap.spi.data.StructuredTable) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) Range(io.cdap.cdap.spi.data.table.field.Range)

Example 22 with Range

use of io.cdap.cdap.spi.data.table.field.Range in project cdap by cdapio.

the class ArtifactStore method clear.

/**
 * Clear all data in the given namespace. Used only in unit tests.
 *
 * @param namespace the namespace to delete data in
 * @throws IOException if there was some problem deleting the data
 */
@VisibleForTesting
void clear(final NamespaceId namespace) throws IOException {
    final Id.Namespace namespaceId = Id.Namespace.fromEntityId(namespace);
    namespacePathLocator.get(namespace).append(ARTIFACTS_PATH).delete(true);
    TransactionRunners.run(transactionRunner, context -> {
        // delete all rows about artifacts in the namespace
        StructuredTable artifactDataTable = getTable(context, StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);
        Range artifactScanRange = createArtifactScanRange(namespace);
        deleteRangeFromTable(artifactDataTable, artifactScanRange);
        // delete all rows about artifacts in the namespace and the plugins they have access to
        StructuredTable pluginDataTable = getTable(context, StoreDefinition.ArtifactStore.PLUGIN_DATA_TABLE);
        Collection<Field<?>> pluginKey = Collections.singleton(Fields.stringField(StoreDefinition.ArtifactStore.PARENT_NAMESPACE_FIELD, namespace.getNamespace()));
        deleteRangeFromTable(pluginDataTable, Range.singleton(pluginKey));
        // delete all rows about universal plugins
        StructuredTable univPluginsDataTable = getTable(context, StoreDefinition.ArtifactStore.UNIV_PLUGIN_DATA_TABLE);
        deleteRangeFromTable(univPluginsDataTable, createUniversalPluginScanRange(namespace.getNamespace(), null));
        // delete app classes in this namespace
        StructuredTable appClassTable = getTable(context, StoreDefinition.ArtifactStore.APP_DATA_TABLE);
        deleteRangeFromTable(appClassTable, createAppClassRange(namespace));
        // delete plugins in this namespace from system artifacts
        // for example, if there was an artifact in this namespace that extends a system artifact
        Collection<Field<?>> systemPluginKey = Collections.singleton(Fields.stringField(StoreDefinition.ArtifactStore.PARENT_NAMESPACE_FIELD, Id.Namespace.SYSTEM.getId()));
        try (CloseableIterator<StructuredRow> iterator = pluginDataTable.scan(Range.singleton(systemPluginKey), Integer.MAX_VALUE)) {
            while (iterator.hasNext()) {
                StructuredRow row = iterator.next();
                // if the plugin artifact is in the namespace we're deleting, delete this column.
                if (namespaceId.getId().equals(row.getString(StoreDefinition.ArtifactStore.ARTIFACT_NAMESPACE_FIELD))) {
                    pluginDataTable.delete(concatFields(PluginKeyPrefix.fromRow(row), ArtifactCell.fromRow(row)));
                }
            }
        }
    }, IOException.class);
}
Also used : Field(io.cdap.cdap.spi.data.table.field.Field) StructuredTable(io.cdap.cdap.spi.data.StructuredTable) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) StructuredTableId(io.cdap.cdap.spi.data.table.StructuredTableId) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) Range(io.cdap.cdap.spi.data.table.field.Range) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 23 with Range

use of io.cdap.cdap.spi.data.table.field.Range in project cdap by cdapio.

the class ArtifactStore method addPluginsInRangeToMap.

private void addPluginsInRangeToMap(final NamespaceId namespace, List<Id.Artifact> parentArtifacts, Iterator<StructuredRow> iterator, SortedMap<ArtifactDescriptor, PluginClass> plugins, @Nullable Predicate<io.cdap.cdap.proto.id.ArtifactId> range, int limit) {
    // if predicate is null,
    // filter out plugins whose artifacts are not in the system namespace and not in this namespace
    range = range != null ? range : input -> NamespaceId.SYSTEM.equals(input.getParent()) || input.getParent().equals(namespace);
    while (iterator.hasNext()) {
        StructuredRow row = iterator.next();
        ImmutablePair<ArtifactDescriptor, PluginData> pluginPair = getPlugin(row, range);
        if (pluginPair == null) {
            continue;
        }
        PluginData pluginData = pluginPair.getSecond();
        // filter out plugins that don't extend this version of the parent artifact
        for (Id.Artifact parentArtifactId : parentArtifacts) {
            if (pluginData.isUsableBy(parentArtifactId.toEntityId()) && isAllowed(pluginData.pluginClass)) {
                plugins.put(pluginPair.getFirst(), pluginData.pluginClass);
                break;
            }
        }
        if (limit < plugins.size()) {
            plugins.remove(plugins.lastKey());
        }
    }
}
Also used : Arrays(java.util.Arrays) ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) TransactionRunners(io.cdap.cdap.spi.data.transaction.TransactionRunners) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Spliterators(java.util.Spliterators) Inject(com.google.inject.Inject) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) ArtifactClasses(io.cdap.cdap.api.artifact.ArtifactClasses) Fields(io.cdap.cdap.spi.data.table.field.Fields) GsonBuilder(com.google.gson.GsonBuilder) Gson(com.google.gson.Gson) ArtifactSortOrder(io.cdap.cdap.proto.artifact.ArtifactSortOrder) Map(java.util.Map) Field(io.cdap.cdap.spi.data.table.field.Field) URI(java.net.URI) Collector(java.util.stream.Collector) StoreDefinition(io.cdap.cdap.store.StoreDefinition) Predicate(java.util.function.Predicate) Collection(java.util.Collection) Set(java.util.Set) ApplicationClass(io.cdap.cdap.api.artifact.ApplicationClass) StructuredTableContext(io.cdap.cdap.spi.data.StructuredTableContext) Collectors(java.util.stream.Collectors) PluginClass(io.cdap.cdap.api.plugin.PluginClass) BinaryOperator(java.util.function.BinaryOperator) Id(io.cdap.cdap.common.id.Id) List(java.util.List) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) Optional(java.util.Optional) Constants(io.cdap.cdap.common.conf.Constants) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) ArtifactAlreadyExistsException(io.cdap.cdap.common.ArtifactAlreadyExistsException) SortedMap(java.util.SortedMap) SchemaTypeAdapter(io.cdap.cdap.internal.io.SchemaTypeAdapter) TransactionException(io.cdap.cdap.spi.data.transaction.TransactionException) Location(org.apache.twill.filesystem.Location) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) PluginNotExistsException(io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException) StructuredTableId(io.cdap.cdap.spi.data.table.StructuredTableId) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) Locations(io.cdap.cdap.common.io.Locations) StreamSupport(java.util.stream.StreamSupport) LinkedHashSet(java.util.LinkedHashSet) Nullable(javax.annotation.Nullable) OutputStream(java.io.OutputStream) Iterator(java.util.Iterator) Files(java.nio.file.Files) Throwables(com.google.common.base.Throwables) Impersonator(io.cdap.cdap.security.impersonation.Impersonator) IOException(java.io.IOException) LocationFactory(org.apache.twill.filesystem.LocationFactory) Maps(com.google.common.collect.Maps) Schema(io.cdap.cdap.api.data.schema.Schema) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) TableNotFoundException(io.cdap.cdap.spi.data.TableNotFoundException) File(java.io.File) MinMaxPriorityQueue(com.google.common.collect.MinMaxPriorityQueue) EntityImpersonator(io.cdap.cdap.security.impersonation.EntityImpersonator) AbstractMap(java.util.AbstractMap) Requirements(io.cdap.cdap.api.plugin.Requirements) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) TreeMap(java.util.TreeMap) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) StructuredTable(io.cdap.cdap.spi.data.StructuredTable) Range(io.cdap.cdap.spi.data.table.field.Range) NamespacePathLocator(io.cdap.cdap.common.namespace.NamespacePathLocator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Collections(java.util.Collections) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) StructuredTableId(io.cdap.cdap.spi.data.table.StructuredTableId) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId)

Example 24 with Range

use of io.cdap.cdap.spi.data.table.field.Range in project cdap by caskdata.

the class ProgramScheduleStoreDataset method deleteSchedules.

/**
 * Removes all schedules for a specific application from the store.
 *
 * @param appId the application id for which to delete the schedules
 * @return the IDs of the schedules that were deleted
 */
// TODO: fix the bug that this method will return fake schedule id https://issues.cask.co/browse/CDAP-13626
public List<ScheduleId> deleteSchedules(ApplicationId appId, long deleteTime) throws IOException {
    List<ScheduleId> deleted = new ArrayList<>();
    Collection<Field<?>> scanKeys = getScheduleKeysForApplicationScan(appId);
    Range range = Range.singleton(scanKeys);
    // First collect all the schedules that are going to be deleted
    try (CloseableIterator<StructuredRow> iterator = scheduleStore.scan(range, Integer.MAX_VALUE)) {
        while (iterator.hasNext()) {
            StructuredRow row = iterator.next();
            if (row.getString(StoreDefinition.ProgramScheduleStore.SCHEDULE) != null) {
                markScheduleAsDeleted(row, deleteTime);
                deleted.add(rowToScheduleId(row));
            }
        }
    }
    // Then delete all triggers for the app
    triggerStore.deleteAll(range);
    return deleted;
}
Also used : Field(io.cdap.cdap.spi.data.table.field.Field) ArrayList(java.util.ArrayList) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) ScheduleId(io.cdap.cdap.proto.id.ScheduleId) Range(io.cdap.cdap.spi.data.table.field.Range)

Example 25 with Range

use of io.cdap.cdap.spi.data.table.field.Range in project cdap by caskdata.

the class ProgramScheduleStoreDataset method deleteSchedules.

/**
 * Removes all schedules for a specific program from the store.
 *
 * @param programId the program id for which to delete the schedules
 * @return the IDs of the schedules that were deleted
 */
// TODO: fix the bug that this method will return fake schedule id https://issues.cask.co/browse/CDAP-13626
public List<ScheduleId> deleteSchedules(ProgramId programId, long deleteTime) throws IOException {
    List<ScheduleId> deleted = new ArrayList<>();
    Collection<Field<?>> scanKeys = getScheduleKeysForApplicationScan(programId.getParent());
    Range range = Range.singleton(scanKeys);
    // First collect all the schedules that are going to be deleted
    try (CloseableIterator<StructuredRow> iterator = scheduleStore.scan(range, Integer.MAX_VALUE)) {
        while (iterator.hasNext()) {
            StructuredRow row = iterator.next();
            String serializedSchedule = row.getString(StoreDefinition.ProgramScheduleStore.SCHEDULE);
            if (serializedSchedule != null) {
                ProgramSchedule schedule = GSON.fromJson(serializedSchedule, ProgramSchedule.class);
                if (programId.equals(schedule.getProgramId())) {
                    markScheduleAsDeleted(row, deleteTime);
                    Collection<Field<?>> deleteKeys = getScheduleKeys(row);
                    triggerStore.deleteAll(Range.singleton(deleteKeys));
                    deleted.add(rowToScheduleId(row));
                }
            }
        }
    }
    return deleted;
}
Also used : Field(io.cdap.cdap.spi.data.table.field.Field) ProgramSchedule(io.cdap.cdap.internal.app.runtime.schedule.ProgramSchedule) ArrayList(java.util.ArrayList) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) ScheduleId(io.cdap.cdap.proto.id.ScheduleId) Range(io.cdap.cdap.spi.data.table.field.Range)

Aggregations

Range (io.cdap.cdap.spi.data.table.field.Range)46 StructuredRow (io.cdap.cdap.spi.data.StructuredRow)28 ArrayList (java.util.ArrayList)28 Field (io.cdap.cdap.spi.data.table.field.Field)24 StructuredTable (io.cdap.cdap.spi.data.StructuredTable)18 HashSet (java.util.HashSet)12 LinkedHashSet (java.util.LinkedHashSet)12 Set (java.util.Set)12 IOException (java.io.IOException)10 Collection (java.util.Collection)10 LinkedHashMap (java.util.LinkedHashMap)10 ArtifactRange (io.cdap.cdap.api.artifact.ArtifactRange)8 Map (java.util.Map)8 Nullable (javax.annotation.Nullable)8 VisibleForTesting (com.google.common.annotations.VisibleForTesting)6 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)6 CloseableIterator (io.cdap.cdap.api.dataset.lib.CloseableIterator)6 Fields (io.cdap.cdap.spi.data.table.field.Fields)6 Collections (java.util.Collections)6 List (java.util.List)6