Examples with StructuredRow io.cdap.cdap.spi.data.StructuredRow used on opensource projects

Example 61 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class SystemAppTestBaseTest method testTableOperations.

@Test
public void testTableOperations() throws Exception {
    StructuredTableAdmin tableAdmin = getStructuredTableAdmin();
    StructuredTableId id = new StructuredTableId("t0");
    Assert.assertFalse(tableAdmin.exists(id));
    String keyCol = "key";
    String valCol = "val";
    tableAdmin.create(new StructuredTableSpecification.Builder().withId(id).withFields(new FieldType(keyCol, FieldType.Type.STRING), new FieldType(valCol, FieldType.Type.STRING)).withPrimaryKeys(keyCol).build());
    try {
        TransactionRunner transactionRunner = getTransactionRunner();
        String key = "k0";
        String val = "v0";
        transactionRunner.run(context -> {
            StructuredTable table = context.getTable(id);
            List<Field<?>> fields = new ArrayList<>();
            fields.add(Fields.stringField(keyCol, key));
            Optional<StructuredRow> row = table.read(fields);
            Assert.assertFalse(row.isPresent());
            fields.add(Fields.stringField(valCol, val));
            table.upsert(fields);
        });
        transactionRunner.run(context -> {
            StructuredTable table = context.getTable(id);
            List<Field<?>> keyField = Collections.singletonList(Fields.stringField(keyCol, key));
            Optional<StructuredRow> row = table.read(keyField);
            Assert.assertTrue(row.isPresent());
            Assert.assertEquals(val, row.get().getString(valCol));
        });
    } finally {
        tableAdmin.drop(id);
    }
}

Example 62 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class FileMetadataCleaner method scanFilesToDelete.

@Nullable
@SuppressWarnings("ConstantConditions")
private Range scanFilesToDelete(StructuredTable table, int fileCleanupBatchSize, long tillTime, List<DeletedEntry> toDelete, AtomicReference<Range> range) throws IOException {
    try (CloseableIterator<StructuredRow> iter = table.scan(range.get(), fileCleanupBatchSize)) {
        while (iter.hasNext()) {
            if (toDelete.size() >= fileCleanupBatchSize) {
                return null;
            }
            StructuredRow row = iter.next();
            long creationTime = row.getLong(StoreDefinition.LogFileMetaStore.CREATION_TIME_FIELD);
            if (creationTime <= tillTime) {
                // expired - can be deleted
                toDelete.add(new DeletedEntry(row.getString(StoreDefinition.LogFileMetaStore.LOGGING_CONTEXT_FIELD), row.getLong(StoreDefinition.LogFileMetaStore.EVENT_TIME_FIELD), row.getLong(StoreDefinition.LogFileMetaStore.CREATION_TIME_FIELD), row.getString(StoreDefinition.LogFileMetaStore.FILE_FIELD)));
            } else {
                // return range to skip this logging context and move to next one.
                return Range.from(ImmutableList.of(Fields.stringField(StoreDefinition.LogFileMetaStore.LOGGING_CONTEXT_FIELD, row.getString(StoreDefinition.LogFileMetaStore.LOGGING_CONTEXT_FIELD))), Range.Bound.EXCLUSIVE);
            }
        }
        // if there are no more files to delete, return next range as null.
        return null;
    }
}

Example 63 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class AppMetadataStore method getActiveRuns.

/**
 * Get active runs for the given programs. Active runs means program run with status STARTING, PENDING,
 * RUNNING or SUSPENDED.
 *
 * @param ids set of program ids to fetch for active run records
 * @return a map from {@link ProgramId} to a {@link Collection} of {@link RunRecordDetail}. It is guaranteed to have
 *         an entry for each of the given program id.
 * @throws IOException if failed to fetch the run records.
 */
public Map<ProgramId, Collection<RunRecordDetail>> getActiveRuns(Collection<ProgramId> ids) throws IOException {
    Collection<Range> ranges = new ArrayList<>();
    Map<ProgramId, Collection<RunRecordDetail>> result = new LinkedHashMap<>();
    for (ProgramId programId : ids) {
        ranges.add(Range.singleton(getRunRecordProgramPrefix(TYPE_RUN_RECORD_ACTIVE, programId)));
        result.put(programId, new LinkedHashSet<>());
    }
    try (CloseableIterator<StructuredRow> iterator = getRunRecordsTable().multiScan(ranges, Integer.MAX_VALUE)) {
        while (iterator.hasNext()) {
            StructuredRow row = iterator.next();
            RunRecordDetail meta = deserializeRunRecordMeta(row);
            result.get(meta.getProgramRunId().getParent()).add(meta);
        }
    }
    return result;
}

Example 64 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class AppMetadataStore method scanActiveRuns.

/**
 * Scans active runs, starting from the given cursor.
 *
 * @param cursor the cursor to start the scan. A cursor can be obtained
 *               from the call to the given {@link BiConsumer} for some previous scan, or use
 *               {@link Cursor#EMPTY} to start a scan at the beginning.
 * @param limit maximum number of records to scan
 * @param consumer a {@link BiConsumer} to consume the scan result
 * @throws IOException if failed to query the storage
 */
public void scanActiveRuns(Cursor cursor, int limit, BiConsumer<Cursor, RunRecordDetail> consumer) throws IOException {
    Collection<Field<?>> begin = cursor.fields;
    if (begin.isEmpty()) {
        begin = getRunRecordStatusPrefix(TYPE_RUN_RECORD_ACTIVE);
    }
    Range range = Range.create(begin, cursor.bound, getRunRecordStatusPrefix(TYPE_RUN_RECORD_ACTIVE), Range.Bound.INCLUSIVE);
    StructuredTable table = getRunRecordsTable();
    try (CloseableIterator<StructuredRow> iterator = table.scan(range, limit)) {
        while (iterator.hasNext()) {
            StructuredRow row = iterator.next();
            consumer.accept(new Cursor(row.getPrimaryKeys(), Range.Bound.EXCLUSIVE), deserializeRunRecordMeta(row));
        }
    }
}

Example 65 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class AppMetadataStore method getProgramRunCounts.

/**
 * Get the run counts of the given program collections.
 *
 * @param programIds the collection of program ids to get the program
 * @return the map of the program id to its run count
 */
public Map<ProgramId, Long> getProgramRunCounts(Collection<ProgramId> programIds) throws BadRequestException, IOException {
    if (programIds.size() > 100) {
        throw new BadRequestException(String.format("%d programs found, the maximum number supported is 100", programIds.size()));
    }
    Map<ProgramId, Long> result = programIds.stream().collect(Collectors.toMap(id -> id, id -> 0L, (v1, v2) -> 0L, LinkedHashMap::new));
    List<List<Field<?>>> multiKeys = programIds.stream().map(id -> getProgramCountPrimaryKeys(TYPE_COUNT, id)).collect(Collectors.toList());
    for (StructuredRow row : getProgramCountsTable().multiRead(multiKeys)) {
        ProgramId programId = getApplicationIdFromRow(row).program(ProgramType.valueOf(row.getString(StoreDefinition.AppMetadataStore.PROGRAM_TYPE_FIELD)), row.getString(StoreDefinition.AppMetadataStore.PROGRAM_FIELD));
        result.put(programId, row.getLong(StoreDefinition.AppMetadataStore.COUNTS));
    }
    return result;
}