Examples with StructuredRow io.cdap.cdap.spi.data.StructuredRow used on opensource projects

Example 11 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class ArtifactStore method deleteRangeFromTable.

private void deleteRangeFromTable(StructuredTable table, Range range) throws IOException {
    try (CloseableIterator<StructuredRow> iterator = table.scan(range, Integer.MAX_VALUE)) {
        while (iterator.hasNext()) {
            StructuredRow row = iterator.next();
            table.delete(row.getPrimaryKeys());
        }
    }
}

Example 12 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class ArtifactStore method getPluginsInArtifact.

private SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginsInArtifact(StructuredTable artifactDataTable, Id.Artifact artifactId, Predicate<PluginClass> filter) throws ArtifactNotFoundException, IOException {
    SortedMap<ArtifactDescriptor, Set<PluginClass>> result = new TreeMap<>();
    // Make sure the artifact exists
    ArtifactCell artifactCell = new ArtifactCell(artifactId);
    Optional<StructuredRow> row = artifactDataTable.read(artifactCell.keys);
    if (!row.isPresent()) {
        throw new ArtifactNotFoundException(artifactId.toEntityId());
    }
    // include any plugin classes that are inside the artifact itself and is accepted by the filter
    ArtifactData artifactData = GSON.fromJson(row.get().getString(StoreDefinition.ArtifactStore.ARTIFACT_DATA_FIELD), ArtifactData.class);
    Set<PluginClass> plugins = artifactData.meta.getClasses().getPlugins().stream().filter(filter).collect(Collectors.toCollection(LinkedHashSet::new));
    if (!plugins.isEmpty()) {
        Location location = Locations.getLocationFromAbsolutePath(locationFactory, artifactData.getLocationPath());
        ArtifactDescriptor descriptor = new ArtifactDescriptor(artifactId.getNamespace().getId(), artifactId.toArtifactId(), location);
        result.put(descriptor, plugins);
    }
    return result;
}

Example 13 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class ArtifactStore method getPluginClasses.

/**
 * Get all plugin classes of the given type and name that extend the given parent artifact.
 * Results are returned as a map from plugin artifact to plugins in that artifact.
 *
 * @param namespace the namespace to search for plugins. The system namespace is always included.
 * @param parentArtifactRange the parent artifact range to find plugins for
 * @param type the type of plugin to look for
 * @param name the name of the plugin to look for
 * @param pluginRange the predicate for the plugins
 * @param limit the limit number of the result
 * @param order the order of the result
 * @return an unmodifiable map of plugin artifact to plugin classes of the given type and name, accessible by the
 *         given artifact. The map will never be null, and will never be empty.
 * @throws PluginNotExistsException if no plugin with the given type and name exists in the namespace
 * @throws IOException if there was an exception reading metadata from the metastore
 */
public SortedMap<ArtifactDescriptor, PluginClass> getPluginClasses(NamespaceId namespace, ArtifactRange parentArtifactRange, String type, String name, @Nullable final Predicate<io.cdap.cdap.proto.id.ArtifactId> pluginRange, int limit, ArtifactSortOrder order) throws IOException, ArtifactNotFoundException, PluginNotExistsException {
    SortedMap<ArtifactDescriptor, PluginClass> result = TransactionRunners.run(transactionRunner, context -> {
        StructuredTable artifactDataTable = getTable(context, StoreDefinition.ArtifactStore.ARTIFACT_DATA_TABLE);
        List<ArtifactDetail> parentArtifactDetails = getArtifacts(artifactDataTable, parentArtifactRange, Integer.MAX_VALUE, null);
        if (parentArtifactDetails.isEmpty()) {
            throw new ArtifactNotFoundException(parentArtifactRange.getNamespace(), parentArtifactRange.getName());
        }
        SortedMap<ArtifactDescriptor, PluginClass> plugins = order == ArtifactSortOrder.DESC ? new TreeMap<>(Collections.reverseOrder()) : new TreeMap<>();
        List<Id.Artifact> parentArtifacts = new ArrayList<>();
        for (ArtifactDetail parentArtifactDetail : parentArtifactDetails) {
            parentArtifacts.add(Id.Artifact.from(Id.Namespace.from(parentArtifactRange.getNamespace()), parentArtifactDetail.getDescriptor().getArtifactId()));
            Set<PluginClass> parentPlugins = parentArtifactDetail.getMeta().getClasses().getPlugins();
            for (PluginClass pluginClass : parentPlugins) {
                if (pluginClass.getName().equals(name) && pluginClass.getType().equals(type) && isAllowed(pluginClass)) {
                    plugins.put(parentArtifactDetail.getDescriptor(), pluginClass);
                    break;
                }
            }
        }
        // Add all plugins that extends from the given set of parents
        StructuredTable pluginTable = getTable(context, StoreDefinition.ArtifactStore.PLUGIN_DATA_TABLE);
        PluginKeyPrefix pluginKey = new PluginKeyPrefix(parentArtifactRange.getNamespace(), parentArtifactRange.getName(), type, name);
        try (CloseableIterator<StructuredRow> iterator = pluginTable.scan(Range.singleton(pluginKey.keys), Integer.MAX_VALUE)) {
            addPluginsInRangeToMap(namespace, parentArtifacts, iterator, plugins, pluginRange, limit);
        }
        // Add all universal plugins
        StructuredTable uniPluginTable = getTable(context, StoreDefinition.ArtifactStore.UNIV_PLUGIN_DATA_TABLE);
        for (String ns : Arrays.asList(namespace.getNamespace(), NamespaceId.SYSTEM.getNamespace())) {
            UniversalPluginKeyPrefix universalPluginKey = new UniversalPluginKeyPrefix(ns, type, name);
            try (CloseableIterator<StructuredRow> iterator = uniPluginTable.scan(Range.singleton(universalPluginKey.keys), Integer.MAX_VALUE)) {
                addPluginsInRangeToMap(namespace, parentArtifacts, iterator, plugins, pluginRange, limit);
            }
        }
        return Collections.unmodifiableSortedMap(plugins);
    }, IOException.class, ArtifactNotFoundException.class);
    if (result.isEmpty()) {
        throw new PluginNotExistsException(new NamespaceId(parentArtifactRange.getNamespace()), type, name);
    }
    return result;
}

Example 14 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class AppMetadataStore method scanActiveRuns.

/**
 * Scans active runs, starting from the given cursor.
 *
 * @param cursor the cursor to start the scan. A cursor can be obtained
 *               from the call to the given {@link BiConsumer} for some previous scan, or use
 *               {@link Cursor#EMPTY} to start a scan at the beginning.
 * @param limit maximum number of records to scan
 * @param consumer a {@link BiConsumer} to consume the scan result
 * @throws IOException if failed to query the storage
 */
public void scanActiveRuns(Cursor cursor, int limit, BiConsumer<Cursor, RunRecordDetail> consumer) throws IOException {
    Collection<Field<?>> begin = cursor.fields;
    if (begin.isEmpty()) {
        begin = getRunRecordStatusPrefix(TYPE_RUN_RECORD_ACTIVE);
    }
    Range range = Range.create(begin, cursor.bound, getRunRecordStatusPrefix(TYPE_RUN_RECORD_ACTIVE), Range.Bound.INCLUSIVE);
    StructuredTable table = getRunRecordsTable();
    try (CloseableIterator<StructuredRow> iterator = table.scan(range, limit)) {
        while (iterator.hasNext()) {
            StructuredRow row = iterator.next();
            consumer.accept(new Cursor(row.getPrimaryKeys(), Range.Bound.EXCLUSIVE), deserializeRunRecordMeta(row));
        }
    }
}

Example 15 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class WorkflowTable method getNeighbors.

/**
 * Returns a map of WorkflowRunId to WorkflowRunRecord that are close to the WorkflowRunId provided by the user.
 *
 * @param id The workflow
 * @param runId The runid of the workflow
 * @param limit The limit on each side of the run that we want to see into
 * @param timeInterval The time interval that we want the results to be spaced apart
 * @return A Map of WorkflowRunId to the corresponding Workflow Run Record. A map is used so that duplicates of
 * the WorkflowRunRecord are not obtained
 */
private Map<String, WorkflowRunRecord> getNeighbors(WorkflowId id, RunId runId, int limit, long timeInterval) throws IOException {
    long startTime = RunIds.getTime(runId, TimeUnit.SECONDS);
    Map<String, WorkflowRunRecord> workflowRunRecords = new HashMap<>();
    int i = -limit;
    long prevStartTime = startTime - (limit * timeInterval);
    // The loop iterates across the range that is startTime - (limit * timeInterval) to
    // startTime + (limit * timeInterval) since we want to capture all runs that started in this range.
    // Since we want to stop getting the same key, we have the prevStartTime become 1 more than the time at which
    // the last record was found if the (interval * the count of the loop) is less than the time.
    long upperBound = startTime + (limit * timeInterval);
    while (prevStartTime <= upperBound) {
        List<Field<?>> lowerBoundFields = getPrimaryKeyFields(id, prevStartTime);
        List<Field<?>> upperBoundFields = getPrimaryKeyFields(id, upperBound);
        // last primary key which is numeric.
        try (CloseableIterator<StructuredRow> iterator = table.scan(Range.create(lowerBoundFields, Range.Bound.INCLUSIVE, upperBoundFields, Range.Bound.INCLUSIVE), 1)) {
            if (!iterator.hasNext()) {
                return workflowRunRecords;
            }
            StructuredRow indexRow = iterator.next();
            long timeOfNextRecord = indexRow.getLong(StoreDefinition.WorkflowStore.START_TIME_FIELD);
            workflowRunRecords.put(indexRow.getString(StoreDefinition.WorkflowStore.RUN_ID_FIELD), getRunRecordFromRow(indexRow));
            prevStartTime = startTime + (i * timeInterval) < timeOfNextRecord ? timeOfNextRecord + 1 : startTime + (i * timeInterval);
            i++;
        }
    }
    return workflowRunRecords;
}