Examples with RecordIndexUniquenessViolation com.apple.foundationdb.record.RecordIndexUniquenessViolation used on opensource projects

Example 1 with RecordIndexUniquenessViolation

use of com.apple.foundationdb.record.RecordIndexUniquenessViolation in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method checkUniqueness.

protected <M extends Message> void checkUniqueness(@Nonnull FDBIndexableRecord<M> savedRecord, @Nonnull IndexEntry indexEntry) {
    Tuple valueKey = indexEntry.getKey();
    AsyncIterable<KeyValue> kvs = state.transaction.getRange(state.indexSubspace.range(valueKey));
    Tuple primaryKey = savedRecord.getPrimaryKey();
    final CompletableFuture<Void> checker = state.store.getContext().instrument(FDBStoreTimer.Events.CHECK_INDEX_UNIQUENESS, AsyncUtil.forEach(kvs, kv -> {
        Tuple existingEntry = unpackKey(getIndexSubspace(), kv);
        Tuple existingKey = state.index.getEntryPrimaryKey(existingEntry);
        if (!TupleHelpers.equals(primaryKey, existingKey)) {
            if (state.store.isIndexWriteOnly(state.index)) {
                addUniquenessViolation(valueKey, primaryKey, existingKey);
                addUniquenessViolation(valueKey, existingKey, primaryKey);
            } else {
                throw new RecordIndexUniquenessViolation(state.index, indexEntry, primaryKey, existingKey);
            }
        }
    }, getExecutor()));
    // Add a pre-commit check to prevent accidentally committing and getting into an invalid state.
    state.store.addIndexUniquenessCommitCheck(state.index, checker);
}

Example 2 with RecordIndexUniquenessViolation

use of com.apple.foundationdb.record.RecordIndexUniquenessViolation in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method updateIndexKeys.

@Override
@Nonnull
protected <M extends Message> CompletableFuture<Void> updateIndexKeys(@Nonnull final FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull final List<IndexEntry> indexEntries) {
    final int groupPrefixSize = getGroupingCount();
    final List<CompletableFuture<Void>> futures = unique && !remove ? new ArrayList<>(indexEntries.size()) : null;
    for (IndexEntry indexEntry : indexEntries) {
        final long startTime = System.nanoTime();
        final Tuple groupKey = TupleHelpers.subTuple(indexEntry.getKey(), 0, groupPrefixSize);
        Object positionObject = indexEntry.getKey().get(groupPrefixSize);
        if (positionObject == null) {
            continue;
        }
        if (!(positionObject instanceof Number)) {
            // This should be prevented by the checks in the meta-data builder, but just in case
            throw new RecordCoreException("position field in index entry is not a number").addLogInfo(LogMessageKeys.KEY, indexEntry.getKey(), LogMessageKeys.INDEX_NAME, state.index.getName(), LogMessageKeys.INDEX_TYPE, state.index.getType());
        }
        long position = ((Number) positionObject).longValue();
        final int offset = (int) Math.floorMod(position, (long) entrySize);
        position -= offset;
        final byte[] key = state.indexSubspace.pack(groupKey.add(position));
        // This has to be the same size every time, with all the unset bits, or else it gets truncated.
        // We really could use a new mutation that took a linear bit position to set / clear and only did length extension or something like that.
        final byte[] bitmap = new byte[(entrySize + 7) / 8];
        if (remove) {
            if (state.store.isIndexWriteOnly(state.index)) {
                // If the index isn't built, it's possible this key wasn't reached.
                // So initialize it to zeros (or leave it alone).
                state.transaction.mutate(MutationType.BIT_OR, key, bitmap);
            }
            // Otherwise the entry must already exist for us to be removing it,
            // so there is no danger that this will store all (but one) ones in a new key.
            Arrays.fill(bitmap, (byte) 0xFF);
            bitmap[offset / 8] &= ~(byte) (1 << (offset % 8));
            state.transaction.mutate(MutationType.BIT_AND, key, bitmap);
            Arrays.fill(bitmap, (byte) 0x00);
            state.transaction.mutate(MutationType.COMPARE_AND_CLEAR, key, bitmap);
        } else {
            if (unique) {
                // Snapshot read to see if the bit is already set.
                CompletableFuture<Void> future = state.transaction.snapshot().get(key).thenAccept(existing -> {
                    if (existing != null && (existing[offset / 8] & (byte) (1 << (offset % 8))) != 0) {
                        throw new RecordIndexUniquenessViolation(state.index, indexEntry, savedRecord.getPrimaryKey(), // Unfortunately, we don't know the other key.
                        null);
                    }
                });
                futures.add(future);
                // Then arrange to conflict for the position with any concurrent update.
                final byte[] conflictKey = new Subspace(key).pack(offset);
                state.transaction.addReadConflictKey(conflictKey);
                state.transaction.addWriteConflictKey(conflictKey);
            }
            bitmap[offset / 8] |= (byte) (1 << (offset % 8));
            state.transaction.mutate(MutationType.BIT_OR, key, bitmap);
        }
        if (state.store.getTimer() != null) {
            state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.MUTATE_INDEX_ENTRY, startTime);
        }
    }
    return futures != null ? AsyncUtil.whenAll(futures) : AsyncUtil.DONE;
}

Example 3 with RecordIndexUniquenessViolation

use of com.apple.foundationdb.record.RecordIndexUniquenessViolation in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method markIndexReadable.

/**
 * Marks an index as readable. See the version of
 * {@link #markIndexReadable(String) markIndexReadable()}
 * that takes a {@link String} as a parameter for more details.
 *
 * @param index the index to mark readable
 * @return a future that will either complete exceptionally if the index can not
 * be made readable or will contain <code>true</code> if the store was modified
 * and <code>false</code> otherwise
 */
@Nonnull
public CompletableFuture<Boolean> markIndexReadable(@Nonnull Index index) {
    if (recordStoreStateRef.get() == null) {
        return preloadRecordStoreStateAsync().thenCompose(vignore -> markIndexReadable(index));
    }
    addIndexStateReadConflict(index.getName());
    beginRecordStoreStateWrite();
    boolean haveFuture = false;
    try {
        Transaction tr = ensureContextActive();
        byte[] indexKey = indexStateSubspace().pack(index.getName());
        CompletableFuture<Boolean> future = tr.get(indexKey).thenCompose(previous -> {
            if (previous != null) {
                CompletableFuture<Optional<Range>> builtFuture = firstUnbuiltRange(index);
                CompletableFuture<Optional<RecordIndexUniquenessViolation>> uniquenessFuture = whenAllIndexUniquenessCommitChecks(index).thenCompose(vignore -> scanUniquenessViolations(index, 1).first());
                return CompletableFuture.allOf(builtFuture, uniquenessFuture).thenApply(vignore -> {
                    Optional<Range> firstUnbuilt = context.join(builtFuture);
                    Optional<RecordIndexUniquenessViolation> uniquenessViolation = context.join(uniquenessFuture);
                    if (firstUnbuilt.isPresent()) {
                        throw new IndexNotBuiltException("Attempted to make unbuilt index readable", firstUnbuilt.get(), LogMessageKeys.INDEX_NAME, index.getName(), "unbuiltRangeBegin", ByteArrayUtil2.loggable(firstUnbuilt.get().begin), "unbuiltRangeEnd", ByteArrayUtil2.loggable(firstUnbuilt.get().end), subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.SUBSPACE_KEY, index.getSubspaceKey());
                    } else if (uniquenessViolation.isPresent()) {
                        RecordIndexUniquenessViolation wrapped = new RecordIndexUniquenessViolation("Uniqueness violation when making index readable", uniquenessViolation.get());
                        wrapped.addLogInfo(LogMessageKeys.INDEX_NAME, index.getName(), subspaceProvider.logKey(), subspaceProvider.toString(context));
                        throw wrapped;
                    } else {
                        updateIndexState(index.getName(), indexKey, IndexState.READABLE);
                        clearReadableIndexBuildData(tr, index);
                        return true;
                    }
                });
            } else {
                return AsyncUtil.READY_FALSE;
            }
        }).whenComplete((b, t) -> endRecordStoreStateWrite()).thenApply(this::addRemoveReplacedIndexesCommitCheckIfChanged);
        haveFuture = true;
        return future;
    } finally {
        if (!haveFuture) {
            endRecordStoreStateWrite();
        }
    }
}

Example 4 with RecordIndexUniquenessViolation

use of com.apple.foundationdb.record.RecordIndexUniquenessViolation in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method uniquenessViolations.

@Test
public void uniquenessViolations() throws Exception {
    final String indexName = "MySimpleRecord$num_value_unique";
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        assertThat(recordStore.isIndexWriteOnly(indexName), is(false));
        recordStore.markIndexWriteOnly(indexName).get();
        assertThat(recordStore.isIndexWriteOnly(indexName), is(true));
        context.commit();
    }
    TestRecords1Proto.MySimpleRecord record1 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValueUnique(42).build();
    TestRecords1Proto.MySimpleRecord record2 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1793L).setNumValueUnique(42).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.saveRecord(record1);
        recordStore.saveRecord(record2);
        context.commit();
    }
    Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$num_value_unique");
    // Test scan uniqueness violations.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        RecordCursorIterator<RecordIndexUniquenessViolation> cursor = recordStore.scanUniquenessViolations(index).asIterator();
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation first = cursor.next();
        assertEquals(Tuple.from(42L), first.getIndexEntry().getKey());
        assertEquals(Tuple.from(1066L), first.getPrimaryKey());
        assertEquals(Tuple.from(1793L), first.getExistingKey());
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation second = cursor.next();
        assertEquals(Tuple.from(42L), second.getIndexEntry().getKey());
        assertEquals(Tuple.from(1793L), second.getPrimaryKey());
        assertEquals(Tuple.from(1066L), second.getExistingKey());
        assertFalse(cursor.hasNext());
    }
    // Test scan uniqueness violations given value key.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        RecordCursorIterator<RecordIndexUniquenessViolation> cursor = recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(42)).asIterator();
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation first = cursor.next();
        assertEquals(Tuple.from(42L), first.getIndexEntry().getKey());
        assertEquals(Tuple.from(1066L), first.getPrimaryKey());
        assertEquals(Tuple.from(1793L), first.getExistingKey());
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation second = cursor.next();
        assertEquals(Tuple.from(42L), second.getIndexEntry().getKey());
        assertEquals(Tuple.from(1793L), second.getPrimaryKey());
        assertEquals(Tuple.from(1066L), second.getExistingKey());
        assertFalse(cursor.hasNext());
    }
    // Test requesting to resolve uniqueness violations with a remaining record.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.resolveUniquenessViolation(index, Tuple.from(42), Tuple.from(1066L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNotNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
    }
    // Test requesting to resolve uniqueness violations with no remaining record.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.resolveUniquenessViolation(index, Tuple.from(42), null).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
    }
    // Test manually resolving uniqueness violations by deleting one record.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.deleteRecordAsync(Tuple.from(1793L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNotNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
    }
}

Example 5 with RecordIndexUniquenessViolation

use of com.apple.foundationdb.record.RecordIndexUniquenessViolation in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method uniquenessViolationsWithFanOut.

@Test
public void uniquenessViolationsWithFanOut() throws Exception {
    Index index = new Index("MySimpleRecord$repeater", field("repeater", FanType.FanOut), Index.EMPTY_VALUE, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", index);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertThat(recordStore.isIndexWriteOnly(index), is(false));
        recordStore.markIndexWriteOnly(index).get();
        assertThat(recordStore.isIndexWriteOnly(index), is(true));
        context.commit();
    }
    TestRecords1Proto.MySimpleRecord record1 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).addRepeater(1).addRepeater(2).addRepeater(3).build();
    TestRecords1Proto.MySimpleRecord record2 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1793L).addRepeater(2).addRepeater(3).build();
    TestRecords1Proto.MySimpleRecord record3 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1849L).addRepeater(3).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.saveRecord(record1);
        recordStore.saveRecord(record2);
        recordStore.saveRecord(record3);
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(5, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(1)).getCount().get());
        assertEquals(2, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(2)).getCount().get());
        assertEquals(3, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(3)).getCount().get());
        RecordCursorIterator<RecordIndexUniquenessViolation> cursor = recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(3)).asIterator();
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation next = cursor.next();
        assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1066L), next.getPrimaryKey());
        assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1793L), Tuple.from(1849L))));
        assertTrue(cursor.hasNext());
        next = cursor.next();
        assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1793L), next.getPrimaryKey());
        assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1066L), Tuple.from(1849L))));
        assertTrue(cursor.hasNext());
        next = cursor.next();
        assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1849L), next.getPrimaryKey());
        assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1066L), Tuple.from(1793L))));
        assertFalse(cursor.hasNext());
        cursor = recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(2)).asIterator();
        assertTrue(cursor.hasNext());
        next = cursor.next();
        assertEquals(Tuple.from(2L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1066L), next.getPrimaryKey());
        assertEquals(Tuple.from(1793L), next.getExistingKey());
        assertTrue(cursor.hasNext());
        next = cursor.next();
        assertEquals(Tuple.from(2L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1793L), next.getPrimaryKey());
        assertEquals(Tuple.from(1066L), next.getExistingKey());
        assertFalse(cursor.hasNext());
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.resolveUniquenessViolation(index, Key.Evaluated.scalar(3), null).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
        assertNull(recordStore.loadRecord(Tuple.from(1849L)));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1066L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNotNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
        assertNull(recordStore.loadRecord(Tuple.from(1849L)));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1793L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNotNull(recordStore.loadRecord(Tuple.from(1793L)));
        assertNull(recordStore.loadRecord(Tuple.from(1849L)));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1849L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
        assertNotNull(recordStore.loadRecord(Tuple.from(1849L)));
    }
}