Search in sources :

Example 1 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class Main method main.

public static void main(String[] args) {
    // Get a database connection.
    FDBDatabase fdb = FDBDatabaseFactory.instance().getDatabase();
    // Create a subspace using the key space API to create a subspace within
    // the cluster used by this record store. The key space API in general
    // allows the user to specify a hierarchical structure of named sub-paths.
    // Each record store can then fill in the named entries within the path
    // with values relevant to that store. If the key space includes a directory
    // layer directory, then the value supplied by the user will be replaced
    // by a short prefix supplied by the the directory layer. The results from
    // the directory layer are cached locally by the Record Layer to avoid excessive
    // database reads.
    // 
    // In this case, the key space implies that there are multiple "applications"
    // that might be defined to run on the same FoundationDB cluster, and then
    // each "application" might have multiple "environments". This could be used,
    // for example, to connect to either the "prod" or "qa" environment for the same
    // application from within the same code base.
    final KeySpace keySpace = new KeySpace(new DirectoryLayerDirectory("application").addSubdirectory(new KeySpaceDirectory("environment", KeySpaceDirectory.KeyType.STRING)));
    // Create a path for the "record-layer-sample" application's demo environment.
    // Clear all existing data and then return the subspace associated with the key space path.
    final KeySpacePath path = keySpace.path("application", "record-layer-sample").add("environment", "demo");
    // Clear out any data that may be in the record store.
    LOGGER.info("Clearing the Record Store ...");
    fdb.run(context -> {
        path.deleteAllData(context);
        return null;
    });
    // Build the metadata. This simple approach only works for primary
    // keys and secondary indexes defined in the Protobuf message types.
    RecordMetaData rmd = RecordMetaData.build(SampleProto.getDescriptor());
    FDBRecordStore.Builder recordStoreBuilder = FDBRecordStore.newBuilder().setMetaDataProvider(rmd).setKeySpacePath(path);
    // Write records for Vendor and Item.
    LOGGER.info("Writing Vendor and Item record ...");
    fdb.run((FDBRecordContext cx) -> {
        FDBRecordStore store = recordStoreBuilder.copyBuilder().setContext(cx).create();
        store.saveRecord(SampleProto.Vendor.newBuilder().setVendorId(9375L).setVendorName("Acme").build());
        store.saveRecord(SampleProto.Vendor.newBuilder().setVendorId(1066L).setVendorName("Buy n Large").build());
        store.saveRecord(SampleProto.Item.newBuilder().setItemId(4836L).setItemName("GPS").setVendorId(9375L).build());
        store.saveRecord(SampleProto.Item.newBuilder().setItemId(9970L).setItemName("Personal Transport").setVendorId(1066L).build());
        store.saveRecord(SampleProto.Item.newBuilder().setItemId(8380L).setItemName("Piles of Garbage").setVendorId(1066L).build());
        return null;
    });
    // Use the primary key declared in the Vendor message type to read a
    // record.
    LOGGER.info("Reading Vendor record with primary key 9375L ...");
    SampleProto.Vendor.Builder readBuilder = fdb.run((FDBRecordContext cx) -> {
        FDBRecordStore store = recordStoreBuilder.copyBuilder().setContext(cx).open();
        return SampleProto.Vendor.newBuilder().mergeFrom(store.loadRecord(Key.Evaluated.scalar(9375L).toTuple()).getRecord());
    });
    LOGGER.info("    Result -> Id: {}, Name: {}", readBuilder.getVendorId(), readBuilder.getVendorName());
    // Using the secondary index declared in the message type, query
    // Item by vendor ID, then look up the item ID.
    LOGGER.info("Looking for item IDs with vendor ID 9375L ...");
    ArrayList<Long> ids = fdb.run((FDBRecordContext cx) -> {
        ArrayList<Long> itemIDs = new ArrayList<>();
        FDBRecordStore store = recordStoreBuilder.copyBuilder().setContext(cx).open();
        RecordQuery query = RecordQuery.newBuilder().setRecordType("Item").setFilter(Query.field("vendor_id").equalsValue(9375L)).build();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = store.executeQuery(query)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result;
            do {
                result = cursor.getNext();
                if (result.hasNext()) {
                    itemIDs.add(SampleProto.Item.newBuilder().mergeFrom(result.get().getRecord()).getItemId());
                }
            } while (result.hasNext());
        }
        return itemIDs;
    });
    ids.forEach((Long res) -> LOGGER.info("    Result -> Vendor ID: 9375, Item ID: {}", res));
    // A kind of hand-crafted "cross-table join" (in some sense). This returns a list
    // linking the name of each vendor to the names of the products they sell.
    // Note that this query is entirely non-blocking until the end.
    // In SQL, this might look something like:
    // 
    // SELECT Vendor.name, Item.name FROM Item JOIN Vendor ON Vendor.vendor_id = Item.vid
    // 
    // One difference is that the above SQL query will flatten the results out so that there
    // is exactly one returned row per item name (per vendor) where as the map returned by
    // this RecordLayer query will feature exactly one entry per vendor where the key is the
    // vendor name and the value is the vendor's items.
    // 
    // Note that this query is not particularly efficient as is. To make this efficient, one
    // might consider an index on vendor name. This could scan the index to get the vendor
    // name of the Vendor record type and a second index on item by vendor ID, perhaps with
    // the item name in the value portion of the index definition. This would allow the
    // query to be satisfied with one scan of the vendor name index and another scan of the
    // item's vendor ID index (one scan per vendor).
    LOGGER.info("Grouping items by vendor ...");
    Map<String, List<String>> namesToItems = fdb.run((FDBRecordContext cx) -> cx.asyncToSync(FDBStoreTimer.Waits.WAIT_EXECUTE_QUERY, recordStoreBuilder.copyBuilder().setContext(cx).openAsync().thenCompose(store -> {
        // Outer plan gets all of the vendors
        RecordQueryPlan outerPlan = store.planQuery(RecordQuery.newBuilder().setRecordType("Vendor").setRequiredResults(Arrays.asList(field("vendor_id"), field("vendor_name"))).build());
        // Inner plan gets all items for the given vendor ID.
        // Using "equalsParameter" does the plan once and re-uses the plan for each vendor ID.
        RecordQueryPlan innerPlan = store.planQuery(RecordQuery.newBuilder().setRecordType("Item").setRequiredResults(Collections.singletonList(field("item_name"))).setFilter(Query.field("vendor_id").equalsParameter("vid")).build());
        return store.executeQuery(outerPlan).mapPipelined(record -> {
            SampleProto.Vendor vendor = SampleProto.Vendor.newBuilder().mergeFrom(record.getRecord()).build();
            return innerPlan.execute(store, EvaluationContext.forBinding("vid", vendor.getVendorId())).map(innerRecord -> SampleProto.Item.newBuilder().mergeFrom(innerRecord.getRecord()).getItemName()).asList().thenApply(list -> Pair.of(vendor.getVendorName(), list));
        }, 10).asList().thenApply((List<Pair<String, List<String>>> list) -> list.stream().collect(Collectors.toMap(Pair::getKey, Pair::getValue)));
    })));
    namesToItems.forEach((String name, List<String> items) -> LOGGER.info("    Result -> Vendor Name: {}, Item names: {}", name, items));
    // Richer indexes:
    // To build richer primary keys or secondary indexes (than those definable in the protobuf
    // message types), you need to use the more verbose and powerful RecordMetaDataBuilder.
    RecordMetaDataBuilder rmdBuilder = RecordMetaData.newBuilder().setRecords(SampleProto.getDescriptor());
    // Order customers by last name, then first name, then their ID if otherwise equal.
    // NOTE: This operation is dangerous if you have existing data! Existing records are *not*
    // automatically migrated.
    rmdBuilder.getRecordType("Customer").setPrimaryKey(concatenateFields("last_name", "first_name", "customer_id"));
    // Add a global count index. Most record stores should probably add this index as it allows
    // the database to make intelligent decisions based on the current size of the record store.
    rmdBuilder.addUniversalIndex(new Index("globalCount", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
    // Add a FanType.FanOut secondary index for email_address, so that
    // each value for email_address generates its own key in the index.
    rmdBuilder.addIndex("Customer", new Index("email_address", field("email_address", FanType.FanOut), IndexTypes.VALUE));
    // Add a FanType.Concatenate secondary index for preference_tag, so
    // that all values for preference_tag generate a single key in the index.
    rmdBuilder.addIndex("Customer", new Index("preference_tag", field("preference_tag", FanType.Concatenate), IndexTypes.VALUE));
    // Add an index on the count of each preference tag. This allows us to
    // quickly get the number of users for each preference tag. The key
    // provided will create a separate "count" field for each value of the
    // preference_tag field and keep track of the number of customer
    // records with each value.
    rmdBuilder.addIndex("Customer", new Index("preference_tag_count", new GroupingKeyExpression(field("preference_tag", FanType.FanOut), 0), IndexTypes.COUNT));
    // Add a nested secondary index for order such that each value for
    // quantity in Order generates a single key in the index.
    rmdBuilder.addIndex("Customer", new Index("order", field("order", FanType.FanOut).nest("quantity"), IndexTypes.VALUE));
    // Add an index on the sum of the quantity of each item in each
    // order. This can be used to know how many of each item have been ordered across
    // all customers. The grouping key here is a little hairy, but it
    // specifies that the "item_id" column should be used as a grouping key
    // and the quantity used as the sum value, so it will keep track of the
    // quantity ordered of each item.
    rmdBuilder.addIndex("Customer", new Index("item_quantity_sum", new GroupingKeyExpression(field("order", FanType.FanOut).nest(concatenateFields("item_id", "quantity")), 1), IndexTypes.SUM));
    // Rebuild the metadata for the newly added indexes before reading or
    // writing more data.
    RecordMetaData rmd2 = rmdBuilder.getRecordMetaData();
    recordStoreBuilder.setMetaDataProvider(rmd2);
    // Calling "open" on an existing record store with new meta-data will
    // create the index and place them in a "disabled" mode that means that
    // they cannot yet be used for queries. (In particular, the query planner
    // will ignore this index and any attempt to read from the index will
    // throw an error.) To enable querying, one must invoke the online index
    // builder. This will scan through the record store across multiple
    // transactions and populate the new indexes with data from the existing
    // entries. During the build job, the record store remains available for
    // reading and writing, but there may be additional conflicts if the index
    // build job and normal operations happen to mutate the same records.
    RecordStoreState storeState = fdb.run(cx -> {
        FDBRecordStore store = recordStoreBuilder.copyBuilder().setContext(cx).open();
        return store.getRecordStoreState();
    });
    LOGGER.info("Running index builds of new indexes:");
    // Build all of the indexes in parallel by firing off a future for each and
    // then wait for all of them.
    fdb.asyncToSync(null, FDBStoreTimer.Waits.WAIT_ONLINE_BUILD_INDEX, AsyncUtil.whenAll(storeState.getDisabledIndexNames().stream().map(indexName -> {
        // Build this index. It will begin the background job and return a future
        // that will complete when the index is ready for querying.
        OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setRecordStoreBuilder(recordStoreBuilder).setIndex(indexName).build();
        return indexBuilder.buildIndexAsync().thenRun(() -> LOGGER.info("  Index build of {} is complete.", indexName)).whenComplete((vignore, eignore) -> indexBuilder.close());
    }).collect(Collectors.toList())));
    // Write larger records for Customer (and Order).
    LOGGER.info("Adding records with new secondary indexes ...");
    fdb.run((FDBRecordContext cx) -> {
        FDBRecordStore store = recordStoreBuilder.copyBuilder().setContext(cx).open();
        store.saveRecord(SampleProto.Customer.newBuilder().setCustomerId(9264L).setFirstName("John").setLastName("Smith").addEmailAddress("jsmith@example.com").addEmailAddress("john_smith@example.com").addPreferenceTag("books").addPreferenceTag("movies").addOrder(SampleProto.Order.newBuilder().setOrderId(3875L).setItemId(9374L).setQuantity(2)).addOrder(SampleProto.Order.newBuilder().setOrderId(4828L).setItemId(2740L).setQuantity(1)).setPhoneNumber("(703) 555-8255").build());
        store.saveRecord(SampleProto.Customer.newBuilder().setCustomerId(8365L).setFirstName("Jane").setLastName("Doe").addEmailAddress("jdoe@example.com").addEmailAddress("jane_doe@example.com").addPreferenceTag("games").addPreferenceTag("lawn").addPreferenceTag("books").addOrder(SampleProto.Order.newBuilder().setOrderId(9280L).setItemId(2740L).setQuantity(3)).setPhoneNumber("(408) 555-0248").build());
        return null;
    });
    // Get the record count. This uses the global count index to get the
    // full number of records in the store.
    Long recordCount = fdb.run((FDBRecordContext cx) -> cx.asyncToSync(FDBStoreTimer.Waits.WAIT_EXECUTE_QUERY, recordStoreBuilder.copyBuilder().setContext(cx).openAsync().thenCompose(FDBRecordStore::getSnapshotRecordCount)));
    LOGGER.info("Store contains {} records.", recordCount);
    // Query all records with the first name "Jane".
    // Performs a full scan of the primary key index.
    LOGGER.info("Retrieving all customers with first name \"Jane\"...");
    List<String> names = fdb.run((FDBRecordContext cx) -> {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("Customer").setFilter(Query.field("first_name").equalsValue("Jane")).build();
        return readNames(recordStoreBuilder, cx, query);
    });
    names.forEach((String res) -> LOGGER.info("    Result -> {}", res));
    // Query all records with last name "Doe".
    // Scans only the customers from the primary key index.
    LOGGER.info("Retrieving all customers with last name \"Doe\"...");
    names = fdb.run((FDBRecordContext cx) -> {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("Customer").setFilter(Query.field("last_name").equalsValue("Doe")).build();
        return readNames(recordStoreBuilder, cx, query);
    });
    names.forEach((String res) -> LOGGER.info("    Result -> {}", res));
    // Query all records with first_name "Jane" and last_name "Doe"
    // Scans only the customers from the primary key index.
    LOGGER.info("Retrieving all customers with name \"Jane Doe\"...");
    names = fdb.run((FDBRecordContext cx) -> {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("Customer").setFilter(Query.and(Query.field("first_name").equalsValue("Jane"), Query.field("last_name").equalsValue("Doe"))).build();
        return readNames(recordStoreBuilder, cx, query);
    });
    names.forEach((String res) -> LOGGER.info("    Result -> {}", res));
    // Query all records with an email address beginning with "john".
    // Uses FanType.FanOut secondary index.
    LOGGER.info("Retrieving all customers with an email address beginning with \"john\"...");
    Map<String, List<String>> addresses = fdb.run((FDBRecordContext cx) -> {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("Customer").setFilter(Query.field("email_address").oneOfThem().startsWith("john")).build();
        return cx.asyncToSync(FDBStoreTimer.Waits.WAIT_EXECUTE_QUERY, recordStoreBuilder.copyBuilder().setContext(cx).openAsync().thenCompose(store -> {
            Map<String, List<String>> addressMap = new HashMap<>();
            return store.executeQuery(query).forEach((FDBQueriedRecord<Message> record) -> {
                SampleProto.Customer.Builder builder = SampleProto.Customer.newBuilder().mergeFrom(record.getRecord());
                addressMap.put(builder.getFirstName() + " " + builder.getLastName(), builder.getEmailAddressList());
            }).thenApply(v -> addressMap);
        }));
    });
    addresses.forEach((String k, List<String> vals) -> LOGGER.info("    Result -> {} with emails {}", k, vals));
    // Query all records with preference_tags "books" and "movies".
    // Uses FanType.Concatenate secondary index.
    LOGGER.info("Retrieving all customers with preference tags \"books\" and \"movies\"...");
    names = fdb.run((FDBRecordContext cx) -> {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("Customer").setFilter(Query.and(Query.field("preference_tag").oneOfThem().equalsValue("books"), Query.field("preference_tag").oneOfThem().equalsValue("movies"))).build();
        return readNames(recordStoreBuilder, cx, query);
    });
    names.forEach((String res) -> LOGGER.info("    Result -> {}", res));
    // Get the number of customers who have "books" listed as one of their preference tags
    Long bookPreferenceCount = fdb.run((FDBRecordContext cx) -> cx.asyncToSync(FDBStoreTimer.Waits.WAIT_EXECUTE_QUERY, recordStoreBuilder.copyBuilder().setContext(cx).openAsync().thenCompose(store -> {
        Index index = store.getRecordMetaData().getIndex("preference_tag_count");
        IndexAggregateFunction function = new IndexAggregateFunction(FunctionNames.COUNT, index.getRootExpression(), index.getName());
        return store.evaluateAggregateFunction(Collections.singletonList("Customer"), function, Key.Evaluated.scalar("books"), IsolationLevel.SERIALIZABLE).thenApply(tuple -> tuple.getLong(0));
    })));
    LOGGER.info("Number of customers with the \"books\" preference tag: {}", bookPreferenceCount);
    // Query all customers with an order of quantity greater than 2.
    // Uses nested secondary index.
    LOGGER.info("Retrieving all customers with an order of quantity greater than 2 ...");
    names = fdb.run((FDBRecordContext cx) -> {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("Customer").setFilter(Query.field("order").oneOfThem().matches(Query.field("quantity").greaterThan(2))).build();
        return readNames(recordStoreBuilder, cx, query);
    });
    names.forEach((String res) -> LOGGER.info("    Result -> {}", res));
    // Get the sum of the quantity of items ordered for item ID 2740.
    // Using the index, it can determine this by reading a single
    // key in the database.
    Long itemQuantitySum = fdb.run((FDBRecordContext cx) -> cx.asyncToSync(FDBStoreTimer.Waits.WAIT_EXECUTE_QUERY, recordStoreBuilder.copyBuilder().setContext(cx).openAsync().thenCompose(store -> {
        Index index = store.getRecordMetaData().getIndex("item_quantity_sum");
        IndexAggregateFunction function = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
        return store.evaluateAggregateFunction(Collections.singletonList("Customer"), function, Key.Evaluated.scalar(2740L), IsolationLevel.SERIALIZABLE).thenApply(tuple -> tuple.getLong(0));
    })));
    LOGGER.info("Total quantity ordered of item 2740L: {}", itemQuantitySum);
    // Get the sum of the quantity of all items ordered.
    // Using the index, it will do a scan that will hit one key
    // for each unique item id with a single range scan.
    Long allItemsQuantitySum = fdb.run((FDBRecordContext cx) -> cx.asyncToSync(FDBStoreTimer.Waits.WAIT_EXECUTE_QUERY, recordStoreBuilder.copyBuilder().setContext(cx).openAsync().thenCompose(store -> {
        Index index = store.getRecordMetaData().getIndex("item_quantity_sum");
        IndexAggregateFunction function = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
        return store.evaluateAggregateFunction(Collections.singletonList("Customer"), function, TupleRange.ALL, IsolationLevel.SERIALIZABLE).thenApply(tuple -> tuple.getLong(0));
    })));
    LOGGER.info("Total quantity ordered of all items: {}", allItemsQuantitySum);
}
Also used : FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) KeySpace(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpace) LoggerFactory(org.slf4j.LoggerFactory) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) HashMap(java.util.HashMap) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ArrayList(java.util.ArrayList) Key(com.apple.foundationdb.record.metadata.Key) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) Pair(org.apache.commons.lang3.tuple.Pair) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Map(java.util.Map) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) DirectoryLayerDirectory(com.apple.foundationdb.record.provider.foundationdb.keyspace.DirectoryLayerDirectory) Query(com.apple.foundationdb.record.query.expressions.Query) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) KeySpaceDirectory(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) OnlineIndexer(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer) FDBDatabase(com.apple.foundationdb.record.provider.foundationdb.FDBDatabase) FanType(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBDatabaseFactory(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Collections(java.util.Collections) ArrayList(java.util.ArrayList) KeySpace(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpace) Index(com.apple.foundationdb.record.metadata.Index) DirectoryLayerDirectory(com.apple.foundationdb.record.provider.foundationdb.keyspace.DirectoryLayerDirectory) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) ArrayList(java.util.ArrayList) List(java.util.List) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) Pair(org.apache.commons.lang3.tuple.Pair) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) OnlineIndexer(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) KeySpaceDirectory(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) FDBDatabase(com.apple.foundationdb.record.provider.foundationdb.FDBDatabase) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) HashMap(java.util.HashMap) Map(java.util.Map)

Example 2 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectory method listSubdirectoryAsync.

@Nonnull
// SonarQube doesn't realize that the cursor is wrapped and returned
@SuppressWarnings("squid:S2095")
protected RecordCursor<ResolvedKeySpacePath> listSubdirectoryAsync(@Nullable KeySpacePath listFrom, @Nonnull FDBRecordContext context, @Nonnull String subdirName, @Nullable ValueRange<?> valueRange, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (listFrom != null && listFrom.getDirectory() != this) {
        throw new RecordCoreException("Provided path does not belong to this directory").addLogInfo("path", listFrom, "directory", this.getName());
    }
    final KeySpaceDirectory subdir = getSubdirectory(subdirName);
    final CompletableFuture<ResolvedKeySpacePath> resolvedFromFuture = listFrom == null ? CompletableFuture.completedFuture(null) : listFrom.toResolvedPathAsync(context);
    // The chained cursor cannot implement reverse scan, so we implement it by having the
    // inner key value cursor do the reversing but telling the chained cursor we are moving
    // forward.
    final ScanProperties chainedCursorScanProperties;
    if (scanProperties.isReverse()) {
        chainedCursorScanProperties = scanProperties.setReverse(false);
    } else {
        chainedCursorScanProperties = scanProperties;
    }
    // For the read of the individual row keys, we only want to read a single key. In addition,
    // the ChainedCursor is going to do counting of our reads to apply any limits that were specified
    // on the ScanProperties.  We don't want the inner KeyValueCursor in nextTuple() to ALSO count those
    // same reads so we clear out its limits.
    final ScanProperties keyReadScanProperties = scanProperties.with(props -> props.clearState().setReturnedRowLimit(1));
    return new LazyCursor<>(resolvedFromFuture.thenCompose(resolvedFrom -> {
        final Subspace subspace = resolvedFrom == null ? new Subspace() : resolvedFrom.toSubspace();
        return subdir.getValueRange(context, valueRange, subspace).thenApply(range -> {
            final RecordCursor<Tuple> cursor = new ChainedCursor<>(context, lastKey -> nextTuple(context, subspace, range, lastKey, keyReadScanProperties), Tuple::pack, Tuple::fromBytes, continuation, chainedCursorScanProperties);
            return cursor.mapPipelined(tuple -> {
                final Tuple key = Tuple.fromList(tuple.getItems());
                return findChildForKey(context, resolvedFrom, key, 1, 0);
            }, 1);
        });
    }), context.getExecutor());
}
Also used : Arrays(java.util.Arrays) KeyRange(com.apple.foundationdb.record.KeyRange) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LazyCursor(com.apple.foundationdb.record.cursors.LazyCursor) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) Tuple(com.apple.foundationdb.tuple.Tuple) EndpointType(com.apple.foundationdb.record.EndpointType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ScanProperties(com.apple.foundationdb.record.ScanProperties) Map(java.util.Map) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ChainedCursor(com.apple.foundationdb.record.cursors.ChainedCursor) StringWriter(java.io.StringWriter) IOException(java.io.IOException) UUID(java.util.UUID) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Objects(java.util.Objects) KeyValueCursor(com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor) List(java.util.List) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Writer(java.io.Writer) RecordCursor(com.apple.foundationdb.record.RecordCursor) ValueRange(com.apple.foundationdb.record.ValueRange) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) BitSet(java.util.BitSet) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) LazyCursor(com.apple.foundationdb.record.cursors.LazyCursor) RecordCursor(com.apple.foundationdb.record.RecordCursor) ScanProperties(com.apple.foundationdb.record.ScanProperties) Subspace(com.apple.foundationdb.subspace.Subspace) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 3 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class LocatableResolver method resolveWithMetadata.

/**
 * Map the String <code>name</code> to a {@link ResolverResult} within the scope of the path that this object was
 * constructed with. If we are creating the entry for <code>name</code>, The {@link ResolverCreateHooks} provided
 * as <code>hooks</code> will be run. Any metadata that was already present or created can be seen by calling
 * {@link ResolverResult#getMetadata()} on the returned result.
 * Will return the value that's persisted in FDB or create it if it does not exist.
 *
 * @param timer the {@link FDBStoreTimer} used for collecting metrics
 * @param name the value to resolve
 * @param hooks {@link ResolverCreateHooks} to run on create
 * @return a future for the {@link ResolverResult} containing the resolved value and metadata
 */
@Nonnull
public CompletableFuture<ResolverResult> resolveWithMetadata(@Nullable FDBStoreTimer timer, @Nonnull String name, @Nonnull ResolverCreateHooks hooks) {
    final FDBRecordContext context = database.openContext(null, timer);
    boolean started = false;
    try {
        CompletableFuture<ResolverResult> future = resolveWithMetadata(context, name, hooks);
        started = true;
        return future.whenComplete((valIgnore, errIgnore) -> context.close());
    } finally {
        if (!started) {
            context.close();
        }
    }
}
Also used : FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Nonnull(javax.annotation.Nonnull)

Example 4 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class ReadVersionRecordStoreStateCache method get.

@Nonnull
@Override
public CompletableFuture<FDBRecordStoreStateCacheEntry> get(@Nonnull FDBRecordStore recordStore, @Nonnull FDBRecordStoreBase.StoreExistenceCheck existenceCheck) {
    final FDBRecordContext context = recordStore.getContext();
    validateContext(context);
    if (context.hasDirtyStoreState()) {
        // If a store info header has been modified during the course of this transaction's run,
        // don't go to the cache as its results may be stale.
        context.increment(FDBStoreTimer.Counts.STORE_STATE_CACHE_MISS);
        return FDBRecordStoreStateCacheEntry.load(recordStore, existenceCheck);
    } else {
        return context.getReadVersionAsync().thenCompose(readVersion -> {
            final SubspaceProvider subspaceProvider = recordStore.getSubspaceProvider();
            CompletableFuture<FDBRecordStoreStateCacheEntry> storeStateFuture = cache.orElseGet(Pair.of(subspaceProvider, readVersion), () -> {
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug(KeyValueLogMessage.of("store state cache miss", subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.READ_VERSION, readVersion));
                }
                context.increment(FDBStoreTimer.Counts.STORE_STATE_CACHE_MISS);
                return FDBRecordStoreStateCacheEntry.load(recordStore, existenceCheck);
            }).thenCompose(storeState -> storeState.handleCachedState(context, existenceCheck).thenApply(vignore -> storeState));
            if (context.getTimer() != null && MoreAsyncUtil.isCompletedNormally(storeStateFuture)) {
                context.increment(FDBStoreTimer.Counts.STORE_STATE_CACHE_HIT);
            }
            return storeStateFuture;
        });
    }
}
Also used : MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) Logger(org.slf4j.Logger) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) SubspaceProvider(com.apple.foundationdb.record.provider.foundationdb.SubspaceProvider) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) LoggerFactory(org.slf4j.LoggerFactory) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) CompletableFuture(java.util.concurrent.CompletableFuture) FDBDatabase(com.apple.foundationdb.record.provider.foundationdb.FDBDatabase) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) AsyncLoadingCache(com.apple.foundationdb.record.AsyncLoadingCache) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) Pair(org.apache.commons.lang3.tuple.Pair) API(com.apple.foundationdb.annotation.API) Nonnull(javax.annotation.Nonnull) FDBRecordStoreBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) SubspaceProvider(com.apple.foundationdb.record.provider.foundationdb.SubspaceProvider) Nonnull(javax.annotation.Nonnull)

Example 5 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class ScopedInterningLayer method readReverse.

@Override
protected CompletableFuture<Optional<String>> readReverse(FDBStoreTimer timer, Long value) {
    FDBRecordContext context = database.openContext();
    context.setTimer(timer);
    return interningLayerFuture.thenCompose(layer -> layer.readReverse(context, value)).whenComplete((ignored, th) -> context.close());
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ResolverResult(com.apple.foundationdb.record.provider.foundationdb.keyspace.ResolverResult) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) CompletableFuture(java.util.concurrent.CompletableFuture) FDBDatabase(com.apple.foundationdb.record.provider.foundationdb.FDBDatabase) LocatableResolver(com.apple.foundationdb.record.provider.foundationdb.keyspace.LocatableResolver) Subspace(com.apple.foundationdb.subspace.Subspace) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ResolvedKeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.ResolvedKeySpacePath) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)

Aggregations

FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)542 Test (org.junit.jupiter.api.Test)365 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)226 RecordQuery (com.apple.foundationdb.record.query.RecordQuery)215 Message (com.google.protobuf.Message)187 FDBQueriedRecord (com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord)170 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)165 Tuple (com.apple.foundationdb.tuple.Tuple)147 Tag (org.junit.jupiter.api.Tag)136 Tags (com.apple.test.Tags)129 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)123 List (java.util.List)121 Index (com.apple.foundationdb.record.metadata.Index)119 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)114 ArrayList (java.util.ArrayList)112 Collections (java.util.Collections)100 Query (com.apple.foundationdb.record.query.expressions.Query)97 RecordQueryPlanner (com.apple.foundationdb.record.query.plan.RecordQueryPlanner)96 Arrays (java.util.Arrays)94 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)93