use of com.apple.foundationdb.tuple.ByteArrayUtil2 in project fdb-record-layer by FoundationDB.
the class FDBRecordStore method deserializeRecord.
private <M extends Message> CompletableFuture<FDBStoredRecord<M>> deserializeRecord(@Nonnull RecordSerializer<M> typedSerializer, @Nonnull final FDBRawRecord rawRecord, @Nonnull final RecordMetaData metaData, @Nonnull final Optional<CompletableFuture<FDBRecordVersion>> versionFutureOptional) {
final Tuple primaryKey = rawRecord.getPrimaryKey();
final byte[] serialized = rawRecord.getRawRecord();
try {
final M protoRecord = typedSerializer.deserialize(metaData, primaryKey, rawRecord.getRawRecord(), getTimer());
final RecordType recordType = metaData.getRecordTypeForDescriptor(protoRecord.getDescriptorForType());
countKeysAndValues(FDBStoreTimer.Counts.LOAD_RECORD_KEY, FDBStoreTimer.Counts.LOAD_RECORD_KEY_BYTES, FDBStoreTimer.Counts.LOAD_RECORD_VALUE_BYTES, rawRecord);
final FDBStoredRecordBuilder<M> recordBuilder = FDBStoredRecord.newBuilder(protoRecord).setPrimaryKey(primaryKey).setRecordType(recordType).setSize(rawRecord);
if (rawRecord.hasVersion()) {
// In the current format version, the version should be read along with the version,
// so this should be hit the majority of the time.
recordBuilder.setVersion(rawRecord.getVersion());
return CompletableFuture.completedFuture(recordBuilder.build());
} else if (versionFutureOptional.isPresent()) {
// another read (which has hopefully happened in parallel with the main record read in the background).
return versionFutureOptional.get().thenApply(version -> {
recordBuilder.setVersion(version);
return recordBuilder.build();
});
} else {
// this will return an FDBStoredRecord where the version is unset.
return CompletableFuture.completedFuture(recordBuilder.build());
}
} catch (Exception ex) {
final LoggableException ex2 = new RecordCoreException("Failed to deserialize record", ex);
ex2.addLogInfo(subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.PRIMARY_KEY, primaryKey, LogMessageKeys.META_DATA_VERSION, metaData.getVersion());
if (LOGGER.isDebugEnabled()) {
ex2.addLogInfo("serialized", ByteArrayUtil2.loggable(serialized));
}
if (LOGGER.isTraceEnabled()) {
ex2.addLogInfo("descriptor", metaData.getUnionDescriptor().getFile().toProto());
}
throw ex2;
}
}
use of com.apple.foundationdb.tuple.ByteArrayUtil2 in project fdb-record-layer by FoundationDB.
the class FDBRecordStore method markIndexReadable.
/**
* Marks an index as readable. See the version of
* {@link #markIndexReadable(String) markIndexReadable()}
* that takes a {@link String} as a parameter for more details.
*
* @param index the index to mark readable
* @return a future that will either complete exceptionally if the index can not
* be made readable or will contain <code>true</code> if the store was modified
* and <code>false</code> otherwise
*/
@Nonnull
public CompletableFuture<Boolean> markIndexReadable(@Nonnull Index index) {
if (recordStoreStateRef.get() == null) {
return preloadRecordStoreStateAsync().thenCompose(vignore -> markIndexReadable(index));
}
addIndexStateReadConflict(index.getName());
beginRecordStoreStateWrite();
boolean haveFuture = false;
try {
Transaction tr = ensureContextActive();
byte[] indexKey = indexStateSubspace().pack(index.getName());
CompletableFuture<Boolean> future = tr.get(indexKey).thenCompose(previous -> {
if (previous != null) {
CompletableFuture<Optional<Range>> builtFuture = firstUnbuiltRange(index);
CompletableFuture<Optional<RecordIndexUniquenessViolation>> uniquenessFuture = whenAllIndexUniquenessCommitChecks(index).thenCompose(vignore -> scanUniquenessViolations(index, 1).first());
return CompletableFuture.allOf(builtFuture, uniquenessFuture).thenApply(vignore -> {
Optional<Range> firstUnbuilt = context.join(builtFuture);
Optional<RecordIndexUniquenessViolation> uniquenessViolation = context.join(uniquenessFuture);
if (firstUnbuilt.isPresent()) {
throw new IndexNotBuiltException("Attempted to make unbuilt index readable", firstUnbuilt.get(), LogMessageKeys.INDEX_NAME, index.getName(), "unbuiltRangeBegin", ByteArrayUtil2.loggable(firstUnbuilt.get().begin), "unbuiltRangeEnd", ByteArrayUtil2.loggable(firstUnbuilt.get().end), subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.SUBSPACE_KEY, index.getSubspaceKey());
} else if (uniquenessViolation.isPresent()) {
RecordIndexUniquenessViolation wrapped = new RecordIndexUniquenessViolation("Uniqueness violation when making index readable", uniquenessViolation.get());
wrapped.addLogInfo(LogMessageKeys.INDEX_NAME, index.getName(), subspaceProvider.logKey(), subspaceProvider.toString(context));
throw wrapped;
} else {
updateIndexState(index.getName(), indexKey, IndexState.READABLE);
clearReadableIndexBuildData(tr, index);
return true;
}
});
} else {
return AsyncUtil.READY_FALSE;
}
}).whenComplete((b, t) -> endRecordStoreStateWrite()).thenApply(this::addRemoveReplacedIndexesCommitCheckIfChanged);
haveFuture = true;
return future;
} finally {
if (!haveFuture) {
endRecordStoreStateWrite();
}
}
}
use of com.apple.foundationdb.tuple.ByteArrayUtil2 in project fdb-record-layer by FoundationDB.
the class OnlineIndexerBuildIndexTest method singleRebuild.
void singleRebuild(@Nonnull List<TestRecords1Proto.MySimpleRecord> records, @Nullable List<TestRecords1Proto.MySimpleRecord> recordsWhileBuilding, int agents, boolean overlap, boolean splitLongRecords, @Nonnull Index index, @Nonnull Runnable beforeBuild, @Nonnull Runnable afterBuild, @Nonnull Runnable afterReadable) {
LOGGER.info(KeyValueLogMessage.of("beginning rebuild test", TestLogMessageKeys.RECORDS, records.size(), LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.AGENTS, agents, TestLogMessageKeys.OVERLAP, overlap, TestLogMessageKeys.SPLIT_LONG_RECORDS, splitLongRecords, TestLogMessageKeys.INDEX, index));
final FDBStoreTimer timer = new FDBStoreTimer();
final FDBRecordStoreTestBase.RecordMetaDataHook onlySplitHook = metaDataBuilder -> {
if (splitLongRecords) {
metaDataBuilder.setSplitLongRecords(true);
metaDataBuilder.removeIndex("MySimpleRecord$str_value_indexed");
}
};
final FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> {
onlySplitHook.apply(metaDataBuilder);
metaDataBuilder.addIndex("MySimpleRecord", index);
};
LOGGER.info(KeyValueLogMessage.of("inserting elements prior to test", TestLogMessageKeys.RECORDS, records.size()));
openSimpleMetaData(onlySplitHook);
try (FDBRecordContext context = openContext()) {
for (TestRecords1Proto.MySimpleRecord record : records) {
// Check presence first to avoid overwriting version information of previously added records.
Tuple primaryKey = Tuple.from(record.getRecNo());
if (recordStore.loadRecord(primaryKey) == null) {
recordStore.saveRecord(record);
}
}
context.commit();
}
LOGGER.info(KeyValueLogMessage.of("running before build for test"));
beforeBuild.run();
openSimpleMetaData(hook);
LOGGER.info(KeyValueLogMessage.of("adding index", TestLogMessageKeys.INDEX, index));
openSimpleMetaData(hook);
final boolean isAlwaysReadable;
try (FDBRecordContext context = openContext()) {
// care of by OnlineIndexer.
if (!safeBuild) {
LOGGER.info(KeyValueLogMessage.of("marking write-only", TestLogMessageKeys.INDEX, index));
recordStore.clearAndMarkIndexWriteOnly(index).join();
}
isAlwaysReadable = safeBuild && recordStore.isIndexReadable(index);
context.commit();
}
LOGGER.info(KeyValueLogMessage.of("creating online index builder", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.RECORD_TYPES, metaData.recordTypesForIndex(index), LogMessageKeys.SUBSPACE, ByteArrayUtil2.loggable(subspace.pack()), LogMessageKeys.LIMIT, 20, TestLogMessageKeys.RECORDS_PER_SECOND, OnlineIndexer.DEFAULT_RECORDS_PER_SECOND * 100));
final OnlineIndexer.Builder builder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).setConfigLoader(old -> {
OnlineIndexer.Config.Builder conf = OnlineIndexer.Config.newBuilder().setMaxLimit(20).setMaxRetries(Integer.MAX_VALUE).setRecordsPerSecond(OnlineIndexer.DEFAULT_RECORDS_PER_SECOND * 100);
if (ThreadLocalRandom.current().nextBoolean()) {
// randomly enable the progress logging to ensure that it doesn't throw exceptions,
// or otherwise disrupt the build.
LOGGER.info("Setting progress log interval");
conf.setProgressLogIntervalMillis(0);
}
return conf.build();
}).setTimer(timer);
if (ThreadLocalRandom.current().nextBoolean()) {
LOGGER.info("Setting priority to DEFAULT");
builder.setPriority(FDBTransactionPriority.DEFAULT);
}
if (fdb.isTrackLastSeenVersion()) {
LOGGER.info("Setting weak read semantics");
builder.setWeakReadSemantics(new FDBDatabase.WeakReadSemantics(0L, Long.MAX_VALUE, true));
}
if (!safeBuild) {
builder.setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfDisabled(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR));
builder.setUseSynchronizedSession(false);
}
try (OnlineIndexer indexBuilder = builder.build()) {
CompletableFuture<Void> buildFuture;
LOGGER.info(KeyValueLogMessage.of("building index", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, agents, LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.OVERLAP, overlap));
if (agents == 1) {
buildFuture = indexBuilder.buildIndexAsync(false);
} else {
if (overlap) {
CompletableFuture<?>[] futures = new CompletableFuture<?>[agents];
for (int i = 0; i < agents; i++) {
final int agent = i;
futures[i] = safeBuild ? indexBuilder.buildIndexAsync(false).exceptionally(exception -> {
// because the other one is already working on building the index.
if (exception.getCause() instanceof SynchronizedSessionLockedException) {
LOGGER.info(KeyValueLogMessage.of("Detected another worker processing this index", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, agent), exception);
return null;
} else {
throw new CompletionException(exception);
}
}) : indexBuilder.buildIndexAsync(false);
}
buildFuture = CompletableFuture.allOf(futures);
} else {
// Safe builds do not support building ranges yet.
assumeFalse(safeBuild);
buildFuture = indexBuilder.buildEndpoints().thenCompose(tupleRange -> {
if (tupleRange != null) {
long start = tupleRange.getLow().getLong(0);
long end = tupleRange.getHigh().getLong(0);
CompletableFuture<?>[] futures = new CompletableFuture<?>[agents];
for (int i = 0; i < agents; i++) {
long itrStart = start + (end - start) / agents * i;
long itrEnd = (i == agents - 1) ? end : start + (end - start) / agents * (i + 1);
LOGGER.info(KeyValueLogMessage.of("building range", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, i, TestLogMessageKeys.BEGIN, itrStart, TestLogMessageKeys.END, itrEnd));
futures[i] = indexBuilder.buildRange(Key.Evaluated.scalar(itrStart), Key.Evaluated.scalar(itrEnd));
}
return CompletableFuture.allOf(futures);
} else {
return AsyncUtil.DONE;
}
});
}
}
if (safeBuild) {
buildFuture = MoreAsyncUtil.composeWhenComplete(buildFuture, (result, ex) -> indexBuilder.checkAnyOngoingOnlineIndexBuildsAsync().thenAccept(Assertions::assertFalse), fdb::mapAsyncToSyncException);
}
if (recordsWhileBuilding != null && recordsWhileBuilding.size() > 0) {
int i = 0;
while (i < recordsWhileBuilding.size()) {
List<TestRecords1Proto.MySimpleRecord> thisBatch = recordsWhileBuilding.subList(i, Math.min(i + 30, recordsWhileBuilding.size()));
fdb.run(context -> {
FDBRecordStore store = recordStore.asBuilder().setContext(context).build();
thisBatch.forEach(store::saveRecord);
return null;
});
i += 30;
}
}
buildFuture.join();
// if a record is added to a range that has already been built, it will not be counted, otherwise,
// it will.
long additionalScans = 0;
if (recordsWhileBuilding != null && recordsWhileBuilding.size() > 0) {
additionalScans += (long) recordsWhileBuilding.size();
}
try (FDBRecordContext context = openContext()) {
IndexBuildState indexBuildState = context.asyncToSync(FDBStoreTimer.Waits.WAIT_GET_INDEX_BUILD_STATE, IndexBuildState.loadIndexBuildStateAsync(recordStore, index));
IndexState indexState = indexBuildState.getIndexState();
if (isAlwaysReadable) {
assertEquals(IndexState.READABLE, indexState);
} else {
assertEquals(IndexState.WRITE_ONLY, indexState);
assertEquals(indexBuilder.getTotalRecordsScanned(), indexBuildState.getRecordsScanned());
// Count index is not defined so we cannot determine the records in total from it.
assertNull(indexBuildState.getRecordsInTotal());
}
}
assertThat(indexBuilder.getTotalRecordsScanned(), allOf(greaterThanOrEqualTo((long) records.size()), lessThanOrEqualTo((long) records.size() + additionalScans)));
}
KeyValueLogMessage msg = KeyValueLogMessage.build("building index - completed", TestLogMessageKeys.INDEX, index);
msg.addKeysAndValues(timer.getKeysAndValues());
LOGGER.info(msg.toString());
LOGGER.info(KeyValueLogMessage.of("running post build checks", TestLogMessageKeys.INDEX, index));
// uses the index in quereis since the index is readable.
if (!isAlwaysReadable) {
afterBuild.run();
}
LOGGER.info(KeyValueLogMessage.of("verifying range set emptiness", TestLogMessageKeys.INDEX, index));
try (FDBRecordContext context = openContext()) {
RangeSet rangeSet = new RangeSet(recordStore.indexRangeSubspace(metaData.getIndex(index.getName())));
System.out.println("Range set for " + records.size() + " records:\n" + rangeSet.rep(context.ensureActive()).join());
if (!isAlwaysReadable) {
assertEquals(Collections.emptyList(), rangeSet.missingRanges(context.ensureActive()).asList().join());
}
context.commit();
}
LOGGER.info(KeyValueLogMessage.of("marking index readable", TestLogMessageKeys.INDEX, index));
try (FDBRecordContext context = openContext()) {
boolean updated = recordStore.markIndexReadable(index).join();
if (isAlwaysReadable) {
assertFalse(updated);
} else {
assertTrue(updated);
}
context.commit();
}
afterReadable.run();
LOGGER.info(KeyValueLogMessage.of("ending rebuild test", TestLogMessageKeys.RECORDS, records.size(), LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.AGENTS, agents, TestLogMessageKeys.OVERLAP, overlap, TestLogMessageKeys.SPLIT_LONG_RECORDS, splitLongRecords, TestLogMessageKeys.INDEX, index));
}
use of com.apple.foundationdb.tuple.ByteArrayUtil2 in project fdb-record-layer by FoundationDB.
the class ResolverMappingReplicator method copyInternal.
private CompletableFuture<Void> copyInternal(@Nonnull final LocatableResolver replica, @Nonnull final LongAccumulator accumulator, @Nonnull final AtomicInteger counter) {
ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(transactionRowLimit).setTimeLimit(transactionTimeLimitMillis).setIsolationLevel(IsolationLevel.SNAPSHOT).build();
final AtomicReference<byte[]> continuation = new AtomicReference<>(null);
return AsyncUtil.whileTrue(() -> {
final FDBRecordContext context = runner.openContext();
return primary.getMappingSubspaceAsync().thenCompose(primaryMappingSubspace -> {
RecordCursor<KeyValue> cursor = KeyValueCursor.Builder.withSubspace(primaryMappingSubspace).setScanProperties(new ScanProperties(executeProperties)).setContext(context).setContinuation(continuation.get()).build();
return cursor.forEachResultAsync(result -> {
KeyValue kv = result.get();
final String mappedString = primaryMappingSubspace.unpack(kv.getKey()).getString(0);
final ResolverResult mappedValue = valueDeserializer.apply(kv.getValue());
accumulator.accumulate(mappedValue.getValue());
counter.incrementAndGet();
return replica.setMapping(context, mappedString, mappedValue);
}).thenCompose(lastResult -> context.commitAsync().thenRun(() -> {
byte[] nextContinuationBytes = lastResult.getContinuation().toBytes();
if (LOGGER.isInfoEnabled()) {
LOGGER.info(KeyValueLogMessage.of("committing batch", LogMessageKeys.SCANNED_SO_FAR, counter.get(), LogMessageKeys.NEXT_CONTINUATION, ByteArrayUtil2.loggable(nextContinuationBytes)));
}
continuation.set(nextContinuationBytes);
})).whenComplete((vignore, eignore) -> cursor.close()).thenApply(vignore -> Objects.nonNull(continuation.get()));
});
}, runner.getExecutor());
}
Aggregations