use of herddb.utils.DataAccessor in project herddb by diennea.
the class DirectMultipleConcurrentUpdatesSuite method performTest.
protected void performTest(boolean useTransactions, long checkPointPeriod, boolean withIndexes, boolean uniqueIndexes) throws Exception {
Path baseDir = folder.newFolder().toPath();
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, checkPointPeriod);
serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
ConcurrentHashMap<String, Long> expectedValue = new ConcurrentHashMap<>();
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForStandaloneBoot();
DBManager manager = server.getManager();
execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", Collections.emptyList());
if (withIndexes) {
if (uniqueIndexes) {
// use n1 + id in order to not have collisions and lock timeouts
execute(manager, "CREATE UNIQUE INDEX theindex ON mytable (n1, id)", Collections.emptyList());
} else {
execute(manager, "CREATE INDEX theindex ON mytable (n1)", Collections.emptyList());
}
}
long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
for (int i = 0; i < TABLESIZE; i++) {
TestUtils.executeUpdate(manager, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", Arrays.asList("test_" + i, 1, 2), new TransactionContext(tx));
expectedValue.put("test_" + i, 1L);
}
TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
try {
List<Future> futures = new ArrayList<>();
AtomicLong updates = new AtomicLong();
AtomicLong skipped = new AtomicLong();
AtomicLong gets = new AtomicLong();
for (int i = 0; i < TABLESIZE * MULTIPLIER; i++) {
futures.add(threadPool.submit(new Runnable() {
@Override
public void run() {
try {
boolean update = ThreadLocalRandom.current().nextBoolean();
int k = ThreadLocalRandom.current().nextInt(TABLESIZE);
long value = ThreadLocalRandom.current().nextInt(TABLESIZE);
long transactionId;
String key = "test_" + k;
Long actual = expectedValue.remove(key);
if (actual == null) {
// another thread working on this entry, skip
skipped.incrementAndGet();
return;
}
if (update) {
updates.incrementAndGet();
DMLStatementExecutionResult updateResult = TestUtils.executeUpdate(manager, "UPDATE mytable set n1=? WHERE id=?", Arrays.asList(value, "test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
long count = updateResult.getUpdateCount();
transactionId = updateResult.transactionId;
if (count <= 0) {
throw new RuntimeException("not updated ?");
}
} else {
gets.incrementAndGet();
DataScanner res = TestUtils.scanKeepReadLocks(manager, "SELECT * FROM mytable where id=?", Arrays.asList("test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
if (!res.hasNext()) {
throw new RuntimeException("not found?");
}
res.close();
transactionId = res.getTransactionId();
// value did not change actually
value = actual;
}
if (useTransactions) {
if (transactionId <= 0) {
throw new RuntimeException("no transaction ?");
}
commitTransaction(manager, TableSpace.DEFAULT, transactionId);
}
expectedValue.put(key, value);
} catch (Exception err) {
throw new RuntimeException(err);
}
}
}));
}
for (Future f : futures) {
f.get();
}
System.out.println("stats::updates:" + updates);
System.out.println("stats::get:" + gets);
assertTrue(updates.get() > 0);
assertTrue(gets.get() > 0);
List<String> erroredKeys = new ArrayList<>();
for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
List<DataAccessor> records;
DataAccessor data;
try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
records = res.consume();
data = records.get(0);
}
assertEquals(1, records.size());
if (!entry.getValue().equals(data.get("n1"))) {
System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
erroredKeys.add(entry.getKey());
}
}
assertTrue(erroredKeys.isEmpty());
TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
System.out.println("stats::tablesize:" + stats.getTablesize());
System.out.println("stats::dirty records:" + stats.getDirtyrecords());
System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
System.out.println("stats::load count:" + stats.getLoadedPagesCount());
System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
// assertTrue(stats.getUnloadedPagesCount() > 0);
assertEquals(TABLESIZE, stats.getTablesize());
} finally {
threadPool.shutdown();
threadPool.awaitTermination(1, TimeUnit.MINUTES);
}
}
// restart and recovery
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
DBManager manager = server.getManager();
List<String> erroredKeys = new ArrayList<>();
for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
List<DataAccessor> records;
DataAccessor data;
try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
records = res.consume();
data = records.get(0);
}
assertEquals(1, records.size());
if (!entry.getValue().equals(data.get("n1"))) {
System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
erroredKeys.add(entry.getKey());
}
}
assertTrue(erroredKeys.isEmpty());
}
}
use of herddb.utils.DataAccessor in project herddb by diennea.
the class TableManager method executeInsertAsync.
private CompletableFuture<StatementExecutionResult> executeInsertAsync(InsertStatement insert, Transaction transaction, StatementEvaluationContext context) {
/*
an insert can succeed only if the row is valid and the "keys" structure does not contain the requested key
the insert will add the row in the 'buffer' without assigning a page to it
locks: the insert uses global 'insert' lock on the table
the insert will update the 'maxKey' for auto_increment primary keys
*/
Bytes key;
byte[] value;
try {
key = Bytes.from_array(insert.getKeyFunction().computeNewValue(null, context, tableContext));
value = insert.getValuesFunction().computeNewValue(new Record(key, null), context, tableContext);
} catch (StatementExecutionException validationError) {
return Futures.exception(validationError);
} catch (Throwable validationError) {
return Futures.exception(new StatementExecutionException(validationError));
}
List<UniqueIndexLockReference> uniqueIndexes = null;
Map<String, AbstractIndexManager> indexes = tableSpaceManager.getIndexesOnTable(table.name);
if (indexes != null || table.foreignKeys != null) {
try {
DataAccessor values = new Record(key, Bytes.from_array(value)).getDataAccessor(table);
if (table.foreignKeys != null) {
for (ForeignKeyDef fk : table.foreignKeys) {
checkForeignKeyConstraintsAsChildTable(fk, values, context, transaction);
}
}
if (indexes != null) {
for (AbstractIndexManager index : indexes.values()) {
if (index.isUnique()) {
Bytes indexKey = RecordSerializer.serializeIndexKey(values, index.getIndex(), index.getColumnNames());
if (uniqueIndexes == null) {
uniqueIndexes = new ArrayList<>(1);
}
uniqueIndexes.add(new UniqueIndexLockReference(index, indexKey));
} else {
RecordSerializer.validateIndexableValue(values, index.getIndex(), index.getColumnNames());
}
}
}
} catch (IllegalArgumentException | herddb.utils.IllegalDataAccessException | StatementExecutionException err) {
if (err instanceof StatementExecutionException) {
return Futures.exception(err);
} else {
return Futures.exception(new StatementExecutionException(err.getMessage(), err));
}
}
}
final long size = DataPage.estimateEntrySize(key, value);
if (size > maxLogicalPageSize) {
return Futures.exception(new RecordTooBigException("New record " + key + " is to big to be inserted: size " + size + ", max size " + maxLogicalPageSize));
}
CompletableFuture<StatementExecutionResult> res = null;
LockHandle lock = null;
try {
lock = lockForWrite(key, transaction);
if (uniqueIndexes != null) {
for (UniqueIndexLockReference uniqueIndexLock : uniqueIndexes) {
AbstractIndexManager index = uniqueIndexLock.indexManager;
LockHandle lockForIndex = lockForWrite(uniqueIndexLock.key, transaction, index.getIndexName(), index.getLockManager());
if (transaction == null) {
uniqueIndexLock.lockHandle = lockForIndex;
}
if (index.valueAlreadyMapped(uniqueIndexLock.key, null)) {
res = Futures.exception(new UniqueIndexContraintViolationException(index.getIndexName(), key, "key " + key + ", already exists in table " + table.name + " on UNIQUE index " + index.getIndexName()));
}
if (res != null) {
break;
}
}
}
} catch (HerdDBInternalException err) {
res = Futures.exception(err);
}
boolean fallbackToUpsert = false;
if (res == null) {
if (transaction != null) {
if (transaction.recordDeleted(table.name, key)) {
// OK, INSERT on a DELETED record inside this transaction
} else if (transaction.recordInserted(table.name, key) != null) {
// ERROR, INSERT on a INSERTED record inside this transaction
res = Futures.exception(new DuplicatePrimaryKeyException(key, "key " + key + ", decoded as " + RecordSerializer.deserializePrimaryKey(key, table) + ", already exists in table " + table.name + " inside transaction " + transaction.transactionId));
} else if (keyToPage.containsKey(key)) {
if (insert.isUpsert()) {
fallbackToUpsert = true;
} else {
res = Futures.exception(new DuplicatePrimaryKeyException(key, "key " + key + ", decoded as " + RecordSerializer.deserializePrimaryKey(key, table) + ", already exists in table " + table.name + " during transaction " + transaction.transactionId));
}
}
} else if (keyToPage.containsKey(key)) {
if (insert.isUpsert()) {
fallbackToUpsert = true;
} else {
res = Futures.exception(new DuplicatePrimaryKeyException(key, "key " + key + ", decoded as " + RecordSerializer.deserializePrimaryKey(key, table) + ", already exists in table " + table.name));
}
}
}
if (res == null) {
LogEntry entry;
if (fallbackToUpsert) {
entry = LogEntryFactory.update(table, key, Bytes.from_array(value), transaction);
} else {
entry = LogEntryFactory.insert(table, key, Bytes.from_array(value), transaction);
}
CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
res = pos.logSequenceNumber.thenApplyAsync((lsn) -> {
apply(pos, entry, false);
return new DMLStatementExecutionResult(entry.transactionId, 1, key, insert.isReturnValues() ? Bytes.from_array(value) : null);
}, tableSpaceManager.getCallbacksExecutor());
}
if (uniqueIndexes != null) {
// TODO: reverse order
for (UniqueIndexLockReference uniqueIndexLock : uniqueIndexes) {
res = releaseWriteLock(res, uniqueIndexLock.lockHandle, uniqueIndexLock.indexManager.getLockManager());
}
}
if (transaction == null) {
res = releaseWriteLock(res, lock);
}
return res;
}
use of herddb.utils.DataAccessor in project herddb by diennea.
the class TableManager method checkForeignKeyConstraintsAsChildTable.
private void checkForeignKeyConstraintsAsChildTable(ForeignKeyDef fk, DataAccessor values, StatementEvaluationContext context, Transaction transaction) throws StatementExecutionException {
// We are creating a SQL query and then using DBManager
// using an SQL query will let us leverage the SQL Planner
// and use the best index to perform the execution
// the SQL Planner will cache the plan, and the plan will also be
// invalidated consistently during DML operations.
String query = childForeignKeyQueries.computeIfAbsent(fk.name, (l -> {
Table parentTable = tableSpaceManager.getTableManagerByUUID(fk.parentTableId).getTable();
// with '*' we are not going to perform projections or copies
StringBuilder q = new StringBuilder("SELECT * FROM ");
q.append(delimit(parentTable.tablespace));
q.append(".");
q.append(delimit(parentTable.name));
q.append(" WHERE ");
for (int i = 0; i < fk.parentTableColumns.length; i++) {
if (i > 0) {
q.append(" AND ");
}
q.append(delimit(fk.parentTableColumns[i]));
q.append("=?");
}
return q.toString();
}));
final List<Object> valuesToMatch = new ArrayList<>(fk.columns.length);
boolean allNulls = true;
for (int i = 0; i < fk.columns.length; i++) {
Object value = values.get(fk.columns[i]);
allNulls = allNulls && value == null;
valuesToMatch.add(value);
}
if (allNulls) {
// all of the values are null, so no check on the parent table
return;
}
TransactionContext tx = transaction != null ? new TransactionContext(transaction.transactionId) : TransactionContext.NO_TRANSACTION;
boolean fkOk;
try (DataScanner scan = tableSpaceManager.getDbmanager().executeSimpleQuery(tableSpaceManager.getTableSpaceName(), query, valuesToMatch, // only one record
1, // keep read locks in TransactionContext
true, tx, null)) {
List<DataAccessor> resultSet = scan.consume();
fkOk = !resultSet.isEmpty();
} catch (DataScannerException err) {
throw new StatementExecutionException(err);
}
if (!fkOk) {
throw new ForeignKeyViolationException(fk.name, "foreignKey " + table.name + "." + fk.name + " violated");
}
}
use of herddb.utils.DataAccessor in project herddb by diennea.
the class TableManager method applyDelete.
private void applyDelete(Bytes key) throws DataStorageManagerException {
/* This could be a normal or a temporary modifiable page */
final Long pageId = keyToPage.remove(key);
if (pageId == null) {
throw new IllegalStateException("corrupted transaction log: key " + key + " is not present in table " + table.tablespace + "." + table.name);
}
if (LOGGER.isLoggable(Level.FINEST)) {
LOGGER.log(Level.FINEST, "Deleted key " + key + " from page " + pageId + " from table " + table.tablespace + "." + table.name);
}
/*
* We'll try to remove the record if in a writable page, otherwise we'll simply set the old page
* as dirty.
*/
final Map<String, AbstractIndexManager> indexes = tableSpaceManager.getIndexesOnTable(table.name);
/*
* When index is enabled we need the old value to update them, we'll force the page load only if that
* record is really needed.
*/
final DataPage page;
final Record previous;
if (indexes == null) {
/* We don't need the page if isn't loaded or isn't a mutable new page */
page = newPages.get(pageId);
if (page != null) {
pageReplacementPolicy.pageHit(page);
previous = page.get(key);
if (previous == null) {
throw new IllegalStateException("corrupted PK: old page " + pageId + " for deleted record at " + key + " was not found in table " + table.tablespace + "." + table.name);
}
} else {
previous = null;
}
} else {
/* We really need the page for update index old values */
page = loadPageToMemory(pageId, false);
previous = page.get(key);
if (previous == null) {
throw new IllegalStateException("corrupted PK: old page " + pageId + " for deleted record at " + key + " was not found in table " + table.tablespace + "." + table.name);
}
}
if (page == null || page.immutable) {
/* Unloaded or immutable, set it as dirty */
pageSet.setPageDirty(pageId, previous);
} else {
/* Mutable page, need to check if still modifiable or already unloaded */
final Lock lock = page.pageLock.readLock();
lock.lock();
try {
if (page.writable) {
/* We can modify the page directly */
page.remove(key);
} else {
/* Unfortunately is not writable (anymore), set it as dirty */
pageSet.setPageDirty(pageId, previous);
}
} finally {
lock.unlock();
}
}
if (indexes != null) {
/* If there are indexes e have already forced a page load and previous record has been loaded */
DataAccessor values = previous.getDataAccessor(table);
for (AbstractIndexManager index : indexes.values()) {
Bytes indexKey = RecordSerializer.serializeIndexKey(values, index.getIndex(), index.getColumnNames());
index.recordDeleted(key, indexKey);
}
}
}
use of herddb.utils.DataAccessor in project herddb by diennea.
the class TableManager method executeUpdateAsync.
private CompletableFuture<StatementExecutionResult> executeUpdateAsync(UpdateStatement update, Transaction transaction, StatementEvaluationContext context) throws StatementExecutionException, DataStorageManagerException {
// LOGGER.log(Level.SEVERE, "executeUpdateAsync, " + update + ", transaction " + transaction);
AtomicInteger updateCount = new AtomicInteger();
Holder<Bytes> lastKey = new Holder<>();
Holder<byte[]> lastValue = new Holder<>();
/*
an update can succeed only if the row is valid, the key is contains in the "keys" structure
the update will simply override the value of the row, assigning a null page to the row
the update can have a 'where' predicate which is to be evaluated against the decoded row, the update will be executed only if the predicate returns boolean 'true' value (CAS operation)
locks: the update uses a lock on the the key
*/
RecordFunction function = update.getFunction();
long transactionId = transaction != null ? transaction.transactionId : 0;
Predicate predicate = update.getPredicate();
Map<String, AbstractIndexManager> indexes = tableSpaceManager.getIndexesOnTable(table.name);
ScanStatement scan = new ScanStatement(table.tablespace, table, predicate);
List<CompletableFuture<PendingLogEntryWork>> writes = new ArrayList<>();
try {
accessTableData(scan, context, new ScanResultOperation() {
@Override
public void accept(Record current, LockHandle lockHandle) throws StatementExecutionException, LogNotAvailableException, DataStorageManagerException {
List<UniqueIndexLockReference> uniqueIndexes = null;
byte[] newValue;
try {
if (childrenTables != null) {
DataAccessor currentValues = current.getDataAccessor(table);
for (Table childTable : childrenTables) {
executeForeignKeyConstraintsAsParentTable(childTable, currentValues, context, transaction, false);
}
}
newValue = function.computeNewValue(current, context, tableContext);
if (indexes != null || table.foreignKeys != null) {
DataAccessor values = new Record(current.key, Bytes.from_array(newValue)).getDataAccessor(table);
if (table.foreignKeys != null) {
for (ForeignKeyDef fk : table.foreignKeys) {
checkForeignKeyConstraintsAsChildTable(fk, values, context, transaction);
}
}
if (indexes != null) {
for (AbstractIndexManager index : indexes.values()) {
if (index.isUnique()) {
Bytes indexKey = RecordSerializer.serializeIndexKey(values, index.getIndex(), index.getColumnNames());
if (uniqueIndexes == null) {
uniqueIndexes = new ArrayList<>(1);
}
UniqueIndexLockReference uniqueIndexLock = new UniqueIndexLockReference(index, indexKey);
uniqueIndexes.add(uniqueIndexLock);
LockHandle lockForIndex = lockForWrite(uniqueIndexLock.key, transaction, index.getIndexName(), index.getLockManager());
if (transaction == null) {
uniqueIndexLock.lockHandle = lockForIndex;
}
if (index.valueAlreadyMapped(indexKey, current.key)) {
throw new UniqueIndexContraintViolationException(index.getIndexName(), indexKey, "Value " + indexKey + " already present in index " + index.getIndexName());
}
} else {
RecordSerializer.validateIndexableValue(values, index.getIndex(), index.getColumnNames());
}
}
}
}
} catch (IllegalArgumentException | herddb.utils.IllegalDataAccessException | StatementExecutionException err) {
locksManager.releaseLock(lockHandle);
StatementExecutionException finalError;
if (!(err instanceof StatementExecutionException)) {
finalError = new StatementExecutionException(err.getMessage(), err);
} else {
finalError = (StatementExecutionException) err;
}
CompletableFuture<PendingLogEntryWork> res = Futures.exception(finalError);
if (uniqueIndexes != null) {
for (UniqueIndexLockReference lock : uniqueIndexes) {
res = releaseWriteLock(res, lock.lockHandle, lock.indexManager.getLockManager());
}
}
writes.add(res);
return;
}
final long size = DataPage.estimateEntrySize(current.key, newValue);
if (size > maxLogicalPageSize) {
locksManager.releaseLock(lockHandle);
writes.add(Futures.exception(new RecordTooBigException("New version of record " + current.key + " is to big to be update: new size " + size + ", actual size " + DataPage.estimateEntrySize(current) + ", max size " + maxLogicalPageSize)));
return;
}
LogEntry entry = LogEntryFactory.update(table, current.key, Bytes.from_array(newValue), transaction);
CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
final List<UniqueIndexLockReference> _uniqueIndexes = uniqueIndexes;
writes.add(pos.logSequenceNumber.thenApply(lsn -> new PendingLogEntryWork(entry, pos, lockHandle, _uniqueIndexes)));
lastKey.value = current.key;
lastValue.value = newValue;
updateCount.incrementAndGet();
}
}, transaction, true, true);
} catch (HerdDBInternalException err) {
LOGGER.log(Level.SEVERE, "bad error during an update", err);
return Futures.exception(err);
}
if (writes.isEmpty()) {
return CompletableFuture.completedFuture(new DMLStatementExecutionResult(transactionId, 0, null, null));
}
if (writes.size() == 1) {
return writes.get(0).whenCompleteAsync((pending, error) -> {
try {
// apply any of the DML operations
if (error == null) {
apply(pending.pos, pending.entry, false);
}
} finally {
releaseMultiplePendingLogEntryWorks(writes);
}
}, tableSpaceManager.getCallbacksExecutor()).thenApply((pending) -> {
return new DMLStatementExecutionResult(transactionId, updateCount.get(), lastKey.value, update.isReturnValues() ? (lastValue.value != null ? Bytes.from_array(lastValue.value) : null) : null);
});
} else {
return Futures.collect(writes).whenCompleteAsync((pendings, error) -> {
try {
// apply any of the DML operations
if (error == null) {
for (PendingLogEntryWork pending : pendings) {
apply(pending.pos, pending.entry, false);
}
}
} finally {
releaseMultiplePendingLogEntryWorks(writes);
}
}, tableSpaceManager.getCallbacksExecutor()).thenApply((pendings) -> {
return new DMLStatementExecutionResult(transactionId, updateCount.get(), lastKey.value, update.isReturnValues() ? (lastValue.value != null ? Bytes.from_array(lastValue.value) : null) : null);
});
}
}
Aggregations