Search in sources :

Example 1 with ByteArrayUtil2

use of com.apple.foundationdb.tuple.ByteArrayUtil2 in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method deserializeRecord.

private <M extends Message> CompletableFuture<FDBStoredRecord<M>> deserializeRecord(@Nonnull RecordSerializer<M> typedSerializer, @Nonnull final FDBRawRecord rawRecord, @Nonnull final RecordMetaData metaData, @Nonnull final Optional<CompletableFuture<FDBRecordVersion>> versionFutureOptional) {
    final Tuple primaryKey = rawRecord.getPrimaryKey();
    final byte[] serialized = rawRecord.getRawRecord();
    try {
        final M protoRecord = typedSerializer.deserialize(metaData, primaryKey, rawRecord.getRawRecord(), getTimer());
        final RecordType recordType = metaData.getRecordTypeForDescriptor(protoRecord.getDescriptorForType());
        countKeysAndValues(FDBStoreTimer.Counts.LOAD_RECORD_KEY, FDBStoreTimer.Counts.LOAD_RECORD_KEY_BYTES, FDBStoreTimer.Counts.LOAD_RECORD_VALUE_BYTES, rawRecord);
        final FDBStoredRecordBuilder<M> recordBuilder = FDBStoredRecord.newBuilder(protoRecord).setPrimaryKey(primaryKey).setRecordType(recordType).setSize(rawRecord);
        if (rawRecord.hasVersion()) {
            // In the current format version, the version should be read along with the version,
            // so this should be hit the majority of the time.
            recordBuilder.setVersion(rawRecord.getVersion());
            return CompletableFuture.completedFuture(recordBuilder.build());
        } else if (versionFutureOptional.isPresent()) {
            // another read (which has hopefully happened in parallel with the main record read in the background).
            return versionFutureOptional.get().thenApply(version -> {
                recordBuilder.setVersion(version);
                return recordBuilder.build();
            });
        } else {
            // this will return an FDBStoredRecord where the version is unset.
            return CompletableFuture.completedFuture(recordBuilder.build());
        }
    } catch (Exception ex) {
        final LoggableException ex2 = new RecordCoreException("Failed to deserialize record", ex);
        ex2.addLogInfo(subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.PRIMARY_KEY, primaryKey, LogMessageKeys.META_DATA_VERSION, metaData.getVersion());
        if (LOGGER.isDebugEnabled()) {
            ex2.addLogInfo("serialized", ByteArrayUtil2.loggable(serialized));
        }
        if (LOGGER.isTraceEnabled()) {
            ex2.addLogInfo("descriptor", metaData.getUnionDescriptor().getFile().toProto());
        }
        throw ex2;
    }
}
Also used : LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) UnaryOperator(java.util.function.UnaryOperator) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RecordSerializer(com.apple.foundationdb.record.provider.common.RecordSerializer) Subspace(com.apple.foundationdb.subspace.Subspace) MutationType(com.apple.foundationdb.MutationType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Map(java.util.Map) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) QueryToKeyMatcher(com.apple.foundationdb.record.query.QueryToKeyMatcher) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Set(java.util.Set) TupleRange(com.apple.foundationdb.record.TupleRange) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) ByteOrder(java.nio.ByteOrder) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) API(com.apple.foundationdb.annotation.API) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RangeSet(com.apple.foundationdb.async.RangeSet) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Supplier(java.util.function.Supplier) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) ArrayList(java.util.ArrayList) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) ParameterRelationshipGraph(com.apple.foundationdb.record.query.ParameterRelationshipGraph) LoggableException(com.apple.foundationdb.util.LoggableException) CloseableAsyncIterator(com.apple.foundationdb.async.CloseableAsyncIterator) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) CursorLimitManager(com.apple.foundationdb.record.cursors.CursorLimitManager) ExecuteState(com.apple.foundationdb.record.ExecuteState) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) DynamicMessageRecordSerializer(com.apple.foundationdb.record.provider.common.DynamicMessageRecordSerializer) SyntheticRecordPlanner(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner) IndexEntry(com.apple.foundationdb.record.IndexEntry) LoggerFactory(org.slf4j.LoggerFactory) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) ByteBuffer(java.nio.ByteBuffer) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Transaction(com.apple.foundationdb.Transaction) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordMetaDataProto(com.apple.foundationdb.record.RecordMetaDataProto) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) AndComponent(com.apple.foundationdb.record.query.expressions.AndComponent) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) AggregateFunctionNotSupportedException(com.apple.foundationdb.record.AggregateFunctionNotSupportedException) RecordTypeKeyComparison(com.apple.foundationdb.record.query.expressions.RecordTypeKeyComparison) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) MutableRecordStoreState(com.apple.foundationdb.record.MutableRecordStoreState) RecordTypeOrBuilder(com.apple.foundationdb.record.metadata.RecordTypeOrBuilder) SyntheticRecordFromStoredRecordPlan(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordFromStoredRecordPlan) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings) Descriptors(com.google.protobuf.Descriptors) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) CursorStreamingMode(com.apple.foundationdb.record.CursorStreamingMode) Key(com.apple.foundationdb.record.metadata.Key) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) Suppliers(com.google.common.base.Suppliers) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) IndexState(com.apple.foundationdb.record.IndexState) StoreRecordFunction(com.apple.foundationdb.record.metadata.StoreRecordFunction) ReadTransaction(com.apple.foundationdb.ReadTransaction) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) FDBRecordStoreStateCache(com.apple.foundationdb.record.provider.foundationdb.storestate.FDBRecordStoreStateCache) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) RecordType(com.apple.foundationdb.record.metadata.RecordType) LoggableException(com.apple.foundationdb.util.LoggableException) Tuple(com.apple.foundationdb.tuple.Tuple) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) LoggableException(com.apple.foundationdb.util.LoggableException) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) AggregateFunctionNotSupportedException(com.apple.foundationdb.record.AggregateFunctionNotSupportedException)

Example 2 with ByteArrayUtil2

use of com.apple.foundationdb.tuple.ByteArrayUtil2 in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method markIndexReadable.

/**
 * Marks an index as readable. See the version of
 * {@link #markIndexReadable(String) markIndexReadable()}
 * that takes a {@link String} as a parameter for more details.
 *
 * @param index the index to mark readable
 * @return a future that will either complete exceptionally if the index can not
 * be made readable or will contain <code>true</code> if the store was modified
 * and <code>false</code> otherwise
 */
@Nonnull
public CompletableFuture<Boolean> markIndexReadable(@Nonnull Index index) {
    if (recordStoreStateRef.get() == null) {
        return preloadRecordStoreStateAsync().thenCompose(vignore -> markIndexReadable(index));
    }
    addIndexStateReadConflict(index.getName());
    beginRecordStoreStateWrite();
    boolean haveFuture = false;
    try {
        Transaction tr = ensureContextActive();
        byte[] indexKey = indexStateSubspace().pack(index.getName());
        CompletableFuture<Boolean> future = tr.get(indexKey).thenCompose(previous -> {
            if (previous != null) {
                CompletableFuture<Optional<Range>> builtFuture = firstUnbuiltRange(index);
                CompletableFuture<Optional<RecordIndexUniquenessViolation>> uniquenessFuture = whenAllIndexUniquenessCommitChecks(index).thenCompose(vignore -> scanUniquenessViolations(index, 1).first());
                return CompletableFuture.allOf(builtFuture, uniquenessFuture).thenApply(vignore -> {
                    Optional<Range> firstUnbuilt = context.join(builtFuture);
                    Optional<RecordIndexUniquenessViolation> uniquenessViolation = context.join(uniquenessFuture);
                    if (firstUnbuilt.isPresent()) {
                        throw new IndexNotBuiltException("Attempted to make unbuilt index readable", firstUnbuilt.get(), LogMessageKeys.INDEX_NAME, index.getName(), "unbuiltRangeBegin", ByteArrayUtil2.loggable(firstUnbuilt.get().begin), "unbuiltRangeEnd", ByteArrayUtil2.loggable(firstUnbuilt.get().end), subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.SUBSPACE_KEY, index.getSubspaceKey());
                    } else if (uniquenessViolation.isPresent()) {
                        RecordIndexUniquenessViolation wrapped = new RecordIndexUniquenessViolation("Uniqueness violation when making index readable", uniquenessViolation.get());
                        wrapped.addLogInfo(LogMessageKeys.INDEX_NAME, index.getName(), subspaceProvider.logKey(), subspaceProvider.toString(context));
                        throw wrapped;
                    } else {
                        updateIndexState(index.getName(), indexKey, IndexState.READABLE);
                        clearReadableIndexBuildData(tr, index);
                        return true;
                    }
                });
            } else {
                return AsyncUtil.READY_FALSE;
            }
        }).whenComplete((b, t) -> endRecordStoreStateWrite()).thenApply(this::addRemoveReplacedIndexesCommitCheckIfChanged);
        haveFuture = true;
        return future;
    } finally {
        if (!haveFuture) {
            endRecordStoreStateWrite();
        }
    }
}
Also used : LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) UnaryOperator(java.util.function.UnaryOperator) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RecordSerializer(com.apple.foundationdb.record.provider.common.RecordSerializer) Subspace(com.apple.foundationdb.subspace.Subspace) MutationType(com.apple.foundationdb.MutationType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Map(java.util.Map) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) QueryToKeyMatcher(com.apple.foundationdb.record.query.QueryToKeyMatcher) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Set(java.util.Set) TupleRange(com.apple.foundationdb.record.TupleRange) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) ByteOrder(java.nio.ByteOrder) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) API(com.apple.foundationdb.annotation.API) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RangeSet(com.apple.foundationdb.async.RangeSet) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Supplier(java.util.function.Supplier) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) ArrayList(java.util.ArrayList) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) ParameterRelationshipGraph(com.apple.foundationdb.record.query.ParameterRelationshipGraph) LoggableException(com.apple.foundationdb.util.LoggableException) CloseableAsyncIterator(com.apple.foundationdb.async.CloseableAsyncIterator) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) CursorLimitManager(com.apple.foundationdb.record.cursors.CursorLimitManager) ExecuteState(com.apple.foundationdb.record.ExecuteState) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) DynamicMessageRecordSerializer(com.apple.foundationdb.record.provider.common.DynamicMessageRecordSerializer) SyntheticRecordPlanner(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner) IndexEntry(com.apple.foundationdb.record.IndexEntry) LoggerFactory(org.slf4j.LoggerFactory) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) ByteBuffer(java.nio.ByteBuffer) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Transaction(com.apple.foundationdb.Transaction) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordMetaDataProto(com.apple.foundationdb.record.RecordMetaDataProto) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) AndComponent(com.apple.foundationdb.record.query.expressions.AndComponent) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) AggregateFunctionNotSupportedException(com.apple.foundationdb.record.AggregateFunctionNotSupportedException) RecordTypeKeyComparison(com.apple.foundationdb.record.query.expressions.RecordTypeKeyComparison) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) MutableRecordStoreState(com.apple.foundationdb.record.MutableRecordStoreState) RecordTypeOrBuilder(com.apple.foundationdb.record.metadata.RecordTypeOrBuilder) SyntheticRecordFromStoredRecordPlan(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordFromStoredRecordPlan) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings) Descriptors(com.google.protobuf.Descriptors) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) CursorStreamingMode(com.apple.foundationdb.record.CursorStreamingMode) Key(com.apple.foundationdb.record.metadata.Key) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) Suppliers(com.google.common.base.Suppliers) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) IndexState(com.apple.foundationdb.record.IndexState) StoreRecordFunction(com.apple.foundationdb.record.metadata.StoreRecordFunction) ReadTransaction(com.apple.foundationdb.ReadTransaction) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) FDBRecordStoreStateCache(com.apple.foundationdb.record.provider.foundationdb.storestate.FDBRecordStoreStateCache) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) Optional(java.util.Optional) TupleRange(com.apple.foundationdb.record.TupleRange) Range(com.apple.foundationdb.Range) Transaction(com.apple.foundationdb.Transaction) ReadTransaction(com.apple.foundationdb.ReadTransaction) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) Nonnull(javax.annotation.Nonnull)

Example 3 with ByteArrayUtil2

use of com.apple.foundationdb.tuple.ByteArrayUtil2 in project fdb-record-layer by FoundationDB.

the class OnlineIndexerBuildIndexTest method singleRebuild.

void singleRebuild(@Nonnull List<TestRecords1Proto.MySimpleRecord> records, @Nullable List<TestRecords1Proto.MySimpleRecord> recordsWhileBuilding, int agents, boolean overlap, boolean splitLongRecords, @Nonnull Index index, @Nonnull Runnable beforeBuild, @Nonnull Runnable afterBuild, @Nonnull Runnable afterReadable) {
    LOGGER.info(KeyValueLogMessage.of("beginning rebuild test", TestLogMessageKeys.RECORDS, records.size(), LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.AGENTS, agents, TestLogMessageKeys.OVERLAP, overlap, TestLogMessageKeys.SPLIT_LONG_RECORDS, splitLongRecords, TestLogMessageKeys.INDEX, index));
    final FDBStoreTimer timer = new FDBStoreTimer();
    final FDBRecordStoreTestBase.RecordMetaDataHook onlySplitHook = metaDataBuilder -> {
        if (splitLongRecords) {
            metaDataBuilder.setSplitLongRecords(true);
            metaDataBuilder.removeIndex("MySimpleRecord$str_value_indexed");
        }
    };
    final FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> {
        onlySplitHook.apply(metaDataBuilder);
        metaDataBuilder.addIndex("MySimpleRecord", index);
    };
    LOGGER.info(KeyValueLogMessage.of("inserting elements prior to test", TestLogMessageKeys.RECORDS, records.size()));
    openSimpleMetaData(onlySplitHook);
    try (FDBRecordContext context = openContext()) {
        for (TestRecords1Proto.MySimpleRecord record : records) {
            // Check presence first to avoid overwriting version information of previously added records.
            Tuple primaryKey = Tuple.from(record.getRecNo());
            if (recordStore.loadRecord(primaryKey) == null) {
                recordStore.saveRecord(record);
            }
        }
        context.commit();
    }
    LOGGER.info(KeyValueLogMessage.of("running before build for test"));
    beforeBuild.run();
    openSimpleMetaData(hook);
    LOGGER.info(KeyValueLogMessage.of("adding index", TestLogMessageKeys.INDEX, index));
    openSimpleMetaData(hook);
    final boolean isAlwaysReadable;
    try (FDBRecordContext context = openContext()) {
        // care of by OnlineIndexer.
        if (!safeBuild) {
            LOGGER.info(KeyValueLogMessage.of("marking write-only", TestLogMessageKeys.INDEX, index));
            recordStore.clearAndMarkIndexWriteOnly(index).join();
        }
        isAlwaysReadable = safeBuild && recordStore.isIndexReadable(index);
        context.commit();
    }
    LOGGER.info(KeyValueLogMessage.of("creating online index builder", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.RECORD_TYPES, metaData.recordTypesForIndex(index), LogMessageKeys.SUBSPACE, ByteArrayUtil2.loggable(subspace.pack()), LogMessageKeys.LIMIT, 20, TestLogMessageKeys.RECORDS_PER_SECOND, OnlineIndexer.DEFAULT_RECORDS_PER_SECOND * 100));
    final OnlineIndexer.Builder builder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).setConfigLoader(old -> {
        OnlineIndexer.Config.Builder conf = OnlineIndexer.Config.newBuilder().setMaxLimit(20).setMaxRetries(Integer.MAX_VALUE).setRecordsPerSecond(OnlineIndexer.DEFAULT_RECORDS_PER_SECOND * 100);
        if (ThreadLocalRandom.current().nextBoolean()) {
            // randomly enable the progress logging to ensure that it doesn't throw exceptions,
            // or otherwise disrupt the build.
            LOGGER.info("Setting progress log interval");
            conf.setProgressLogIntervalMillis(0);
        }
        return conf.build();
    }).setTimer(timer);
    if (ThreadLocalRandom.current().nextBoolean()) {
        LOGGER.info("Setting priority to DEFAULT");
        builder.setPriority(FDBTransactionPriority.DEFAULT);
    }
    if (fdb.isTrackLastSeenVersion()) {
        LOGGER.info("Setting weak read semantics");
        builder.setWeakReadSemantics(new FDBDatabase.WeakReadSemantics(0L, Long.MAX_VALUE, true));
    }
    if (!safeBuild) {
        builder.setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfDisabled(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR));
        builder.setUseSynchronizedSession(false);
    }
    try (OnlineIndexer indexBuilder = builder.build()) {
        CompletableFuture<Void> buildFuture;
        LOGGER.info(KeyValueLogMessage.of("building index", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, agents, LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.OVERLAP, overlap));
        if (agents == 1) {
            buildFuture = indexBuilder.buildIndexAsync(false);
        } else {
            if (overlap) {
                CompletableFuture<?>[] futures = new CompletableFuture<?>[agents];
                for (int i = 0; i < agents; i++) {
                    final int agent = i;
                    futures[i] = safeBuild ? indexBuilder.buildIndexAsync(false).exceptionally(exception -> {
                        // because the other one is already working on building the index.
                        if (exception.getCause() instanceof SynchronizedSessionLockedException) {
                            LOGGER.info(KeyValueLogMessage.of("Detected another worker processing this index", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, agent), exception);
                            return null;
                        } else {
                            throw new CompletionException(exception);
                        }
                    }) : indexBuilder.buildIndexAsync(false);
                }
                buildFuture = CompletableFuture.allOf(futures);
            } else {
                // Safe builds do not support building ranges yet.
                assumeFalse(safeBuild);
                buildFuture = indexBuilder.buildEndpoints().thenCompose(tupleRange -> {
                    if (tupleRange != null) {
                        long start = tupleRange.getLow().getLong(0);
                        long end = tupleRange.getHigh().getLong(0);
                        CompletableFuture<?>[] futures = new CompletableFuture<?>[agents];
                        for (int i = 0; i < agents; i++) {
                            long itrStart = start + (end - start) / agents * i;
                            long itrEnd = (i == agents - 1) ? end : start + (end - start) / agents * (i + 1);
                            LOGGER.info(KeyValueLogMessage.of("building range", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, i, TestLogMessageKeys.BEGIN, itrStart, TestLogMessageKeys.END, itrEnd));
                            futures[i] = indexBuilder.buildRange(Key.Evaluated.scalar(itrStart), Key.Evaluated.scalar(itrEnd));
                        }
                        return CompletableFuture.allOf(futures);
                    } else {
                        return AsyncUtil.DONE;
                    }
                });
            }
        }
        if (safeBuild) {
            buildFuture = MoreAsyncUtil.composeWhenComplete(buildFuture, (result, ex) -> indexBuilder.checkAnyOngoingOnlineIndexBuildsAsync().thenAccept(Assertions::assertFalse), fdb::mapAsyncToSyncException);
        }
        if (recordsWhileBuilding != null && recordsWhileBuilding.size() > 0) {
            int i = 0;
            while (i < recordsWhileBuilding.size()) {
                List<TestRecords1Proto.MySimpleRecord> thisBatch = recordsWhileBuilding.subList(i, Math.min(i + 30, recordsWhileBuilding.size()));
                fdb.run(context -> {
                    FDBRecordStore store = recordStore.asBuilder().setContext(context).build();
                    thisBatch.forEach(store::saveRecord);
                    return null;
                });
                i += 30;
            }
        }
        buildFuture.join();
        // if a record is added to a range that has already been built, it will not be counted, otherwise,
        // it will.
        long additionalScans = 0;
        if (recordsWhileBuilding != null && recordsWhileBuilding.size() > 0) {
            additionalScans += (long) recordsWhileBuilding.size();
        }
        try (FDBRecordContext context = openContext()) {
            IndexBuildState indexBuildState = context.asyncToSync(FDBStoreTimer.Waits.WAIT_GET_INDEX_BUILD_STATE, IndexBuildState.loadIndexBuildStateAsync(recordStore, index));
            IndexState indexState = indexBuildState.getIndexState();
            if (isAlwaysReadable) {
                assertEquals(IndexState.READABLE, indexState);
            } else {
                assertEquals(IndexState.WRITE_ONLY, indexState);
                assertEquals(indexBuilder.getTotalRecordsScanned(), indexBuildState.getRecordsScanned());
                // Count index is not defined so we cannot determine the records in total from it.
                assertNull(indexBuildState.getRecordsInTotal());
            }
        }
        assertThat(indexBuilder.getTotalRecordsScanned(), allOf(greaterThanOrEqualTo((long) records.size()), lessThanOrEqualTo((long) records.size() + additionalScans)));
    }
    KeyValueLogMessage msg = KeyValueLogMessage.build("building index - completed", TestLogMessageKeys.INDEX, index);
    msg.addKeysAndValues(timer.getKeysAndValues());
    LOGGER.info(msg.toString());
    LOGGER.info(KeyValueLogMessage.of("running post build checks", TestLogMessageKeys.INDEX, index));
    // uses the index in quereis since the index is readable.
    if (!isAlwaysReadable) {
        afterBuild.run();
    }
    LOGGER.info(KeyValueLogMessage.of("verifying range set emptiness", TestLogMessageKeys.INDEX, index));
    try (FDBRecordContext context = openContext()) {
        RangeSet rangeSet = new RangeSet(recordStore.indexRangeSubspace(metaData.getIndex(index.getName())));
        System.out.println("Range set for " + records.size() + " records:\n" + rangeSet.rep(context.ensureActive()).join());
        if (!isAlwaysReadable) {
            assertEquals(Collections.emptyList(), rangeSet.missingRanges(context.ensureActive()).asList().join());
        }
        context.commit();
    }
    LOGGER.info(KeyValueLogMessage.of("marking index readable", TestLogMessageKeys.INDEX, index));
    try (FDBRecordContext context = openContext()) {
        boolean updated = recordStore.markIndexReadable(index).join();
        if (isAlwaysReadable) {
            assertFalse(updated);
        } else {
            assertTrue(updated);
        }
        context.commit();
    }
    afterReadable.run();
    LOGGER.info(KeyValueLogMessage.of("ending rebuild test", TestLogMessageKeys.RECORDS, records.size(), LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.AGENTS, agents, TestLogMessageKeys.OVERLAP, overlap, TestLogMessageKeys.SPLIT_LONG_RECORDS, splitLongRecords, TestLogMessageKeys.INDEX, index));
}
Also used : LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RangeSet(com.apple.foundationdb.async.RangeSet) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) ArrayList(java.util.ArrayList) Key(com.apple.foundationdb.record.metadata.Key) Tuple(com.apple.foundationdb.tuple.Tuple) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) SynchronizedSessionLockedException(com.apple.foundationdb.synchronizedsession.SynchronizedSessionLockedException) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) Assumptions.assumeFalse(org.junit.jupiter.api.Assumptions.assumeFalse) Map(java.util.Map) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) Logger(org.slf4j.Logger) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) Matchers.allOf(org.hamcrest.Matchers.allOf) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) IndexState(com.apple.foundationdb.record.IndexState) CompletionException(java.util.concurrent.CompletionException) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Message(com.google.protobuf.Message) Assertions(org.junit.jupiter.api.Assertions) Comparator(java.util.Comparator) Collections(java.util.Collections) IndexState(com.apple.foundationdb.record.IndexState) CompletableFuture(java.util.concurrent.CompletableFuture) SynchronizedSessionLockedException(com.apple.foundationdb.synchronizedsession.SynchronizedSessionLockedException) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) RangeSet(com.apple.foundationdb.async.RangeSet) Assertions(org.junit.jupiter.api.Assertions) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) CompletionException(java.util.concurrent.CompletionException) Tuple(com.apple.foundationdb.tuple.Tuple)

Example 4 with ByteArrayUtil2

use of com.apple.foundationdb.tuple.ByteArrayUtil2 in project fdb-record-layer by FoundationDB.

the class ResolverMappingReplicator method copyInternal.

private CompletableFuture<Void> copyInternal(@Nonnull final LocatableResolver replica, @Nonnull final LongAccumulator accumulator, @Nonnull final AtomicInteger counter) {
    ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(transactionRowLimit).setTimeLimit(transactionTimeLimitMillis).setIsolationLevel(IsolationLevel.SNAPSHOT).build();
    final AtomicReference<byte[]> continuation = new AtomicReference<>(null);
    return AsyncUtil.whileTrue(() -> {
        final FDBRecordContext context = runner.openContext();
        return primary.getMappingSubspaceAsync().thenCompose(primaryMappingSubspace -> {
            RecordCursor<KeyValue> cursor = KeyValueCursor.Builder.withSubspace(primaryMappingSubspace).setScanProperties(new ScanProperties(executeProperties)).setContext(context).setContinuation(continuation.get()).build();
            return cursor.forEachResultAsync(result -> {
                KeyValue kv = result.get();
                final String mappedString = primaryMappingSubspace.unpack(kv.getKey()).getString(0);
                final ResolverResult mappedValue = valueDeserializer.apply(kv.getValue());
                accumulator.accumulate(mappedValue.getValue());
                counter.incrementAndGet();
                return replica.setMapping(context, mappedString, mappedValue);
            }).thenCompose(lastResult -> context.commitAsync().thenRun(() -> {
                byte[] nextContinuationBytes = lastResult.getContinuation().toBytes();
                if (LOGGER.isInfoEnabled()) {
                    LOGGER.info(KeyValueLogMessage.of("committing batch", LogMessageKeys.SCANNED_SO_FAR, counter.get(), LogMessageKeys.NEXT_CONTINUATION, ByteArrayUtil2.loggable(nextContinuationBytes)));
                }
                continuation.set(nextContinuationBytes);
            })).whenComplete((vignore, eignore) -> cursor.close()).thenApply(vignore -> Objects.nonNull(continuation.get()));
        });
    }, runner.getExecutor());
}
Also used : IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) KeyValue(com.apple.foundationdb.KeyValue) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) LoggerFactory(org.slf4j.LoggerFactory) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) Objects(java.util.Objects) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyValueCursor(com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) LongAccumulator(java.util.concurrent.atomic.LongAccumulator) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) FDBDatabaseRunner(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseRunner) Nonnull(javax.annotation.Nonnull) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyValue(com.apple.foundationdb.KeyValue) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanProperties(com.apple.foundationdb.record.ScanProperties) AtomicReference(java.util.concurrent.atomic.AtomicReference)

Aggregations

AsyncUtil (com.apple.foundationdb.async.AsyncUtil)4 KeyValueLogMessage (com.apple.foundationdb.record.logging.KeyValueLogMessage)4 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)4 KeyValue (com.apple.foundationdb.KeyValue)3 API (com.apple.foundationdb.annotation.API)3 MoreAsyncUtil (com.apple.foundationdb.async.MoreAsyncUtil)3 RangeSet (com.apple.foundationdb.async.RangeSet)3 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)3 IndexState (com.apple.foundationdb.record.IndexState)3 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)3 RecordCursor (com.apple.foundationdb.record.RecordCursor)3 ScanProperties (com.apple.foundationdb.record.ScanProperties)3 MutationType (com.apple.foundationdb.MutationType)2 Range (com.apple.foundationdb.Range)2 ReadTransaction (com.apple.foundationdb.ReadTransaction)2 Transaction (com.apple.foundationdb.Transaction)2 SpotBugsSuppressWarnings (com.apple.foundationdb.annotation.SpotBugsSuppressWarnings)2 AsyncIterable (com.apple.foundationdb.async.AsyncIterable)2 AsyncIterator (com.apple.foundationdb.async.AsyncIterator)2 CloseableAsyncIterator (com.apple.foundationdb.async.CloseableAsyncIterator)2