Examples with StructuredRow io.cdap.cdap.spi.data.StructuredRow used on opensource projects

Example 36 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class AbstractCheckpointManager method getCheckpoint.

@Override
public Map<Integer, Checkpoint<T>> getCheckpoint(Set<Integer> partitions) throws IOException {
    return TransactionRunners.run(transactionRunner, context -> {
        Map<Integer, Checkpoint<T>> checkpoints = new HashMap<>();
        StructuredTable table = context.getTable(StoreDefinition.LogCheckpointStore.LOG_CHECKPOINT_TABLE);
        for (int partition : partitions) {
            Optional<StructuredRow> optionalRow = table.read(getKeyFields(rowKeyPrefix, partition));
            StructuredRow row = optionalRow.orElse(null);
            checkpoints.put(partition, fromRow(row));
        }
        return checkpoints;
    }, IOException.class);
}

Example 37 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class AbstractCheckpointManager method getCheckpoint.

@Override
public Checkpoint<T> getCheckpoint(int partition) throws IOException {
    Checkpoint<T> checkpoint = TransactionRunners.run(transactionRunner, context -> {
        StructuredTable table = context.getTable(StoreDefinition.LogCheckpointStore.LOG_CHECKPOINT_TABLE);
        Optional<StructuredRow> optionalRow = table.read(getKeyFields(rowKeyPrefix, partition));
        StructuredRow row = optionalRow.orElse(null);
        return fromRow(row);
    }, IOException.class);
    LOG.trace("Read checkpoint {} for partition {}", checkpoint, partition);
    return checkpoint;
}

Example 38 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class FileMetaDataReader method getFiles.

private List<LogLocation> getFiles(StructuredTable metaTable, LogPathIdentifier logPathIdentifier, long endTimestampMs) throws IOException {
    // create scanner with
    // start rowkey prefix:context:event-time(0):create-time(0)
    // end rowkey  prefix:context:event-time(endTimestamp):0(create-time doesn't matter for get files)
    // add these files to the list
    List<LogLocation> files = new ArrayList<>();
    Range scanRange = Range.create(getKeyFields(logPathIdentifier.getRowkey(), 0L, 0L), Range.Bound.INCLUSIVE, getPartialKey(logPathIdentifier.getRowkey(), endTimestampMs), Range.Bound.INCLUSIVE);
    try (CloseableIterator<StructuredRow> iter = metaTable.scan(scanRange, Integer.MAX_VALUE)) {
        while (iter.hasNext()) {
            StructuredRow row = iter.next();
            files.add(fromRow(row, logPathIdentifier.getNamespaceId()));
        }
    }
    return files;
}

Example 39 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by caskdata.

the class FileMetadataCleaner method scanFilesToDelete.

@Nullable
@SuppressWarnings("ConstantConditions")
private Range scanFilesToDelete(StructuredTable table, int fileCleanupBatchSize, long tillTime, List<DeletedEntry> toDelete, AtomicReference<Range> range) throws IOException {
    try (CloseableIterator<StructuredRow> iter = table.scan(range.get(), fileCleanupBatchSize)) {
        while (iter.hasNext()) {
            if (toDelete.size() >= fileCleanupBatchSize) {
                return null;
            }
            StructuredRow row = iter.next();
            long creationTime = row.getLong(StoreDefinition.LogFileMetaStore.CREATION_TIME_FIELD);
            if (creationTime <= tillTime) {
                // expired - can be deleted
                toDelete.add(new DeletedEntry(row.getString(StoreDefinition.LogFileMetaStore.LOGGING_CONTEXT_FIELD), row.getLong(StoreDefinition.LogFileMetaStore.EVENT_TIME_FIELD), row.getLong(StoreDefinition.LogFileMetaStore.CREATION_TIME_FIELD), row.getString(StoreDefinition.LogFileMetaStore.FILE_FIELD)));
            } else {
                // return range to skip this logging context and move to next one.
                return Range.from(ImmutableList.of(Fields.stringField(StoreDefinition.LogFileMetaStore.LOGGING_CONTEXT_FIELD, row.getString(StoreDefinition.LogFileMetaStore.LOGGING_CONTEXT_FIELD))), Range.Bound.EXCLUSIVE);
            }
        }
        // if there are no more files to delete, return next range as null.
        return null;
    }
}

Example 40 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class CapabilityStatusStore method checkAllEnabled.

@Override
public void checkAllEnabled(Collection<String> capabilities) throws IOException, CapabilityNotAvailableException {
    List<List<Field<?>>> multiKeys = new ArrayList<>();
    capabilities.forEach(capability -> multiKeys.add(Collections.singletonList(Fields.stringField(StoreDefinition.CapabilitiesStore.NAME_FIELD, capability))));
    Map<String, String> capabilityMap = TransactionRunners.run(transactionRunner, context -> {
        StructuredTable capabilityTable = context.getTable(StoreDefinition.CapabilitiesStore.CAPABILITIES);
        return capabilityTable.multiRead(multiKeys).stream().collect(Collectors.toMap(structuredRow -> structuredRow.getString(StoreDefinition.CapabilitiesStore.NAME_FIELD), structuredRow -> structuredRow.getString(StoreDefinition.CapabilitiesStore.STATUS_FIELD)));
    }, IOException.class);
    for (String capability : capabilities) {
        if (!capabilityMap.containsKey(capability) || CapabilityStatus.valueOf(capabilityMap.get(capability).toUpperCase()) != CapabilityStatus.ENABLED) {
            throw new CapabilityNotAvailableException(capability);
        }
    }
}