use of herddb.model.Table in project herddb by diennea.
the class RunHerdDB070Test method test.
@Test
public void test() throws Exception {
String file = "herddb.070.joinerror.zip";
File dbdatadir = folder.newFolder("dbdata070_" + file);
try (InputStream in = RunHerdDB070Test.class.getResourceAsStream(file)) {
ZIPUtils.unZip(in, dbdatadir);
}
System.out.println("UNZIPPED TO " + dbdatadir);
final Path dbdata = dbdatadir.toPath().resolve("herddb.070.joinerror").resolve("dbdata");
Path metadataPath = dbdata.resolve("metadata");
Path dataPath = dbdata.resolve("data");
Path logsPath = dbdata.resolve("txlog");
Path tmoDir = dbdata.resolve("tmp");
assertTrue(Files.isDirectory(metadataPath));
assertTrue(Files.isDirectory(dataPath));
assertTrue(Files.isDirectory(logsPath));
Path nodeid = dataPath.resolve("nodeid");
assertTrue(Files.isRegularFile(nodeid));
String id = new String(Files.readAllBytes(nodeid), StandardCharsets.UTF_8);
System.out.println("id:" + id);
String expectedNodeId = "asino";
assertTrue(id.endsWith("\n" + expectedNodeId));
try (DBManager manager = new DBManager(expectedNodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
manager.start();
final String tableSpace = "herd";
final String tableName = "testtable";
assertEquals(expectedNodeId, manager.getNodeId());
assertTrue(manager.waitForTablespace(tableSpace, 10000));
AbstractTableManager tableManagerLicense = manager.getTableSpaceManager("herd").getTableManager("license");
Table tableLicense = tableManagerLicense.getTable();
System.out.println("TABLE PK: " + Arrays.toString(tableLicense.primaryKey));
for (Column c : tableLicense.columns) {
System.out.println("COL: " + c.name + " serialPos: " + c.serialPosition);
}
AbstractTableManager tableManagerCustomer = manager.getTableSpaceManager("herd").getTableManager("customer");
Table tableCustomer = tableManagerCustomer.getTable();
System.out.println("TABLE PK: " + Arrays.toString(tableCustomer.primaryKey));
for (Column c : tableCustomer.columns) {
System.out.println("COL: " + c.name + " serialPos: " + c.serialPosition);
}
{
TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM license", Collections.emptyList(), true, true, false, -1);
System.out.println("TABLE CONTENTS");
try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
List<DataAccessor> consume = scan1.consume();
System.out.println("NUM " + consume.size());
assertEquals(15, consume.size());
for (DataAccessor r : consume) {
System.out.println("RECORD " + r.toMap());
}
}
}
{
TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM customer", Collections.emptyList(), true, true, false, -1);
System.out.println("TABLE CONTENTS");
try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
List<DataAccessor> consume = scan1.consume();
System.out.println("NUM " + consume.size());
assertEquals(7, consume.size());
for (DataAccessor r : consume) {
System.out.println("RECORD " + r.toMap());
}
}
}
{
TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id\n" + " ", Collections.emptyList(), true, true, false, -1);
System.out.println("TABLE CONTENTS");
try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
List<DataAccessor> consume = scan1.consume();
System.out.println("NUM " + consume.size());
assertEquals(9, consume.size());
for (DataAccessor r : consume) {
System.out.println("RECORD " + r.toMap());
}
}
}
{
TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM license t0, customer c WHERE c.customer_id = t0.customer_id", Collections.emptyList(), true, true, false, -1);
System.out.println("TABLE CONTENTS");
try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
List<DataAccessor> consume = scan1.consume();
System.out.println("NUM " + consume.size());
assertEquals(15, consume.size());
for (DataAccessor r : consume) {
System.out.println("RECORD " + r.toMap());
}
}
}
}
}
use of herddb.model.Table in project herddb by diennea.
the class TableManager method scanWithStream.
private DataScanner scanWithStream(ScanStatement statement, StatementEvaluationContext context, Transaction transaction, boolean lockRequired, boolean forWrite) throws StatementExecutionException {
if (transaction != null) {
transaction.increaseRefcount();
}
try {
final TupleComparator comparator = statement.getComparator();
boolean sorted = comparator != null;
boolean sortedByClusteredIndex = comparator != null && comparator.isOnlyPrimaryKeyAndAscending() && keyToPageSortedAscending;
final Projection projection = statement.getProjection();
final boolean applyProjectionDuringScan = projection != null && !sorted;
ScanLimits limits = statement.getLimits();
int maxRows = limits == null ? 0 : limits.computeMaxRows(context);
int offset = limits == null ? 0 : limits.computeOffset(context);
Stream<DataAccessor> result;
Function<Record, DataAccessor> mapper = (Record record) -> {
DataAccessor tuple;
if (applyProjectionDuringScan) {
tuple = projection.map(record.getDataAccessor(table), context);
} else {
tuple = record.getDataAccessor(table);
}
return tuple;
};
Stream<Record> recordsFromTransactionSorted = streamTransactionData(transaction, statement.getPredicate(), context);
Stream<DataAccessor> fromTransactionSorted = recordsFromTransactionSorted != null ? recordsFromTransactionSorted.map(mapper) : null;
if (fromTransactionSorted != null && comparator != null) {
fromTransactionSorted = fromTransactionSorted.sorted(comparator);
}
Stream<DataAccessor> tableData = streamTableData(statement, context, transaction, lockRequired, forWrite).map(mapper);
if (maxRows > 0) {
if (sortedByClusteredIndex) {
// already sorted if needed
if (fromTransactionSorted != null) {
// already sorted from index
tableData = tableData.limit(maxRows + offset);
fromTransactionSorted = fromTransactionSorted.limit(maxRows + offset);
// we need to re-sort after merging the data
result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
} else {
// already sorted from index
tableData = tableData.limit(maxRows + offset);
// no need to re-sort
result = tableData;
}
} else if (sorted) {
// need to sort
tableData = tableData.sorted(comparator);
// already sorted if needed
if (fromTransactionSorted != null) {
tableData = tableData.limit(maxRows + offset);
fromTransactionSorted = fromTransactionSorted.limit(maxRows + offset);
// we need to re-sort after merging the data
result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
} else {
tableData = tableData.limit(maxRows + offset);
// no need to sort again
result = tableData;
}
} else if (fromTransactionSorted == null) {
result = tableData;
} else {
result = Stream.concat(fromTransactionSorted, tableData);
}
} else {
if (sortedByClusteredIndex) {
// already sorted from index
if (fromTransactionSorted != null) {
tableData = tableData.sorted(comparator);
// fromTransactionSorted is already sorted
// we need to re-sort
result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
} else {
result = tableData;
}
} else if (sorted) {
// we need to re-sort
if (fromTransactionSorted != null) {
result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
} else {
result = tableData.sorted(comparator);
}
} else if (fromTransactionSorted != null) {
// no need to sort
result = Stream.concat(fromTransactionSorted, tableData);
} else {
result = tableData;
}
}
if (offset > 0) {
result = result.skip(offset);
}
if (maxRows > 0) {
result = result.limit(maxRows);
}
if (!applyProjectionDuringScan && projection != null) {
result = result.map(r -> projection.map(r, context));
}
String[] fieldNames;
Column[] columns;
if (projection != null) {
fieldNames = projection.getFieldNames();
columns = projection.getColumns();
} else {
fieldNames = table.columnNames;
columns = table.columns;
}
return new StreamDataScanner(transaction, fieldNames, columns, result);
} finally {
if (transaction != null) {
transaction.decreaseRefCount();
}
}
}
use of herddb.model.Table in project herddb by diennea.
the class TableManager method executeUpdateAsync.
private CompletableFuture<StatementExecutionResult> executeUpdateAsync(UpdateStatement update, Transaction transaction, StatementEvaluationContext context) throws StatementExecutionException, DataStorageManagerException {
// LOGGER.log(Level.SEVERE, "executeUpdateAsync, " + update + ", transaction " + transaction);
AtomicInteger updateCount = new AtomicInteger();
Holder<Bytes> lastKey = new Holder<>();
Holder<byte[]> lastValue = new Holder<>();
/*
an update can succeed only if the row is valid, the key is contains in the "keys" structure
the update will simply override the value of the row, assigning a null page to the row
the update can have a 'where' predicate which is to be evaluated against the decoded row, the update will be executed only if the predicate returns boolean 'true' value (CAS operation)
locks: the update uses a lock on the the key
*/
RecordFunction function = update.getFunction();
long transactionId = transaction != null ? transaction.transactionId : 0;
Predicate predicate = update.getPredicate();
Map<String, AbstractIndexManager> indexes = tableSpaceManager.getIndexesOnTable(table.name);
ScanStatement scan = new ScanStatement(table.tablespace, table, predicate);
List<CompletableFuture<PendingLogEntryWork>> writes = new ArrayList<>();
try {
accessTableData(scan, context, new ScanResultOperation() {
@Override
public void accept(Record current, LockHandle lockHandle) throws StatementExecutionException, LogNotAvailableException, DataStorageManagerException {
List<UniqueIndexLockReference> uniqueIndexes = null;
byte[] newValue;
try {
if (childrenTables != null) {
DataAccessor currentValues = current.getDataAccessor(table);
for (Table childTable : childrenTables) {
executeForeignKeyConstraintsAsParentTable(childTable, currentValues, context, transaction, false);
}
}
newValue = function.computeNewValue(current, context, tableContext);
if (indexes != null || table.foreignKeys != null) {
DataAccessor values = new Record(current.key, Bytes.from_array(newValue)).getDataAccessor(table);
if (table.foreignKeys != null) {
for (ForeignKeyDef fk : table.foreignKeys) {
checkForeignKeyConstraintsAsChildTable(fk, values, context, transaction);
}
}
if (indexes != null) {
for (AbstractIndexManager index : indexes.values()) {
if (index.isUnique()) {
Bytes indexKey = RecordSerializer.serializeIndexKey(values, index.getIndex(), index.getColumnNames());
if (uniqueIndexes == null) {
uniqueIndexes = new ArrayList<>(1);
}
UniqueIndexLockReference uniqueIndexLock = new UniqueIndexLockReference(index, indexKey);
uniqueIndexes.add(uniqueIndexLock);
LockHandle lockForIndex = lockForWrite(uniqueIndexLock.key, transaction, index.getIndexName(), index.getLockManager());
if (transaction == null) {
uniqueIndexLock.lockHandle = lockForIndex;
}
if (index.valueAlreadyMapped(indexKey, current.key)) {
throw new UniqueIndexContraintViolationException(index.getIndexName(), indexKey, "Value " + indexKey + " already present in index " + index.getIndexName());
}
} else {
RecordSerializer.validateIndexableValue(values, index.getIndex(), index.getColumnNames());
}
}
}
}
} catch (IllegalArgumentException | herddb.utils.IllegalDataAccessException | StatementExecutionException err) {
locksManager.releaseLock(lockHandle);
StatementExecutionException finalError;
if (!(err instanceof StatementExecutionException)) {
finalError = new StatementExecutionException(err.getMessage(), err);
} else {
finalError = (StatementExecutionException) err;
}
CompletableFuture<PendingLogEntryWork> res = Futures.exception(finalError);
if (uniqueIndexes != null) {
for (UniqueIndexLockReference lock : uniqueIndexes) {
res = releaseWriteLock(res, lock.lockHandle, lock.indexManager.getLockManager());
}
}
writes.add(res);
return;
}
final long size = DataPage.estimateEntrySize(current.key, newValue);
if (size > maxLogicalPageSize) {
locksManager.releaseLock(lockHandle);
writes.add(Futures.exception(new RecordTooBigException("New version of record " + current.key + " is to big to be update: new size " + size + ", actual size " + DataPage.estimateEntrySize(current) + ", max size " + maxLogicalPageSize)));
return;
}
LogEntry entry = LogEntryFactory.update(table, current.key, Bytes.from_array(newValue), transaction);
CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
final List<UniqueIndexLockReference> _uniqueIndexes = uniqueIndexes;
writes.add(pos.logSequenceNumber.thenApply(lsn -> new PendingLogEntryWork(entry, pos, lockHandle, _uniqueIndexes)));
lastKey.value = current.key;
lastValue.value = newValue;
updateCount.incrementAndGet();
}
}, transaction, true, true);
} catch (HerdDBInternalException err) {
LOGGER.log(Level.SEVERE, "bad error during an update", err);
return Futures.exception(err);
}
if (writes.isEmpty()) {
return CompletableFuture.completedFuture(new DMLStatementExecutionResult(transactionId, 0, null, null));
}
if (writes.size() == 1) {
return writes.get(0).whenCompleteAsync((pending, error) -> {
try {
// apply any of the DML operations
if (error == null) {
apply(pending.pos, pending.entry, false);
}
} finally {
releaseMultiplePendingLogEntryWorks(writes);
}
}, tableSpaceManager.getCallbacksExecutor()).thenApply((pending) -> {
return new DMLStatementExecutionResult(transactionId, updateCount.get(), lastKey.value, update.isReturnValues() ? (lastValue.value != null ? Bytes.from_array(lastValue.value) : null) : null);
});
} else {
return Futures.collect(writes).whenCompleteAsync((pendings, error) -> {
try {
// apply any of the DML operations
if (error == null) {
for (PendingLogEntryWork pending : pendings) {
apply(pending.pos, pending.entry, false);
}
}
} finally {
releaseMultiplePendingLogEntryWorks(writes);
}
}, tableSpaceManager.getCallbacksExecutor()).thenApply((pendings) -> {
return new DMLStatementExecutionResult(transactionId, updateCount.get(), lastKey.value, update.isReturnValues() ? (lastValue.value != null ? Bytes.from_array(lastValue.value) : null) : null);
});
}
}
use of herddb.model.Table in project herddb by diennea.
the class TableManager method loadPageToMemory.
private DataPage loadPageToMemory(Long pageId, boolean recovery) throws DataStorageManagerException {
DataPage result = pages.get(pageId);
if (result != null) {
pageReplacementPolicy.pageHit(result);
return result;
}
long _start = System.currentTimeMillis();
long _ioAndLock = 0;
BooleanHolder computed = new BooleanHolder(false);
try {
result = pages.computeIfAbsent(pageId, (id) -> {
try {
computed.value = true;
List<Record> page;
maxCurrentPagesLoads.acquireUninterruptibly();
try {
page = dataStorageManager.readPage(tableSpaceUUID, table.uuid, pageId);
} finally {
maxCurrentPagesLoads.release();
}
loadedPagesCount.increment();
return buildImmutableDataPage(pageId, page);
} catch (DataStorageManagerException err) {
throw new RuntimeException(err);
}
});
if (computed.value) {
_ioAndLock = System.currentTimeMillis();
final Page.Metadata unload = pageReplacementPolicy.add(result);
if (unload != null) {
unload.owner.unload(unload.pageId);
}
}
} catch (RuntimeException error) {
if (error.getCause() != null) {
Throwable cause = error.getCause();
if (cause instanceof DataStorageManagerException) {
if (cause instanceof DataPageDoesNotExistException) {
return null;
}
throw (DataStorageManagerException) cause;
}
}
throw new DataStorageManagerException(error);
}
if (computed.value && LOGGER.isLoggable(Level.FINE)) {
long _stop = System.currentTimeMillis();
LOGGER.log(Level.FINE, "table {0}.{1}, loaded {2} records from page {3} in {4} ms, ({5} ms read + plock, {6} ms unlock)", new Object[] { table.tablespace, table.name, result.size(), pageId, (_stop - _start), (_ioAndLock - _start), (_stop - _ioAndLock) });
}
return result;
}
use of herddb.model.Table in project herddb by diennea.
the class SingleTableDumper method acceptTableStatus.
@Override
public void acceptTableStatus(TableStatus tableStatus) {
try {
Table table = tableManager.getTable();
byte[] tableDefinition = table.serialize();
TableManagerStats stats = tableManager.getStats();
List<byte[]> indexes = tableManager.getAvailableIndexes().stream().map(Index::serialize).collect(Collectors.toList());
long id = channel.generateRequestId();
try (Pdu pdu = channel.sendMessageWithPduReply(id, PduCodec.TablespaceDumpData.write(id, tableSpaceName, dumpId, "beginTable", tableDefinition, stats.getTablesize(), tableStatus.sequenceNumber.ledgerId, tableStatus.sequenceNumber.offset, indexes, null), timeout)) {
}
} catch (InterruptedException | TimeoutException err) {
throw new HerdDBInternalException(err);
}
}
Aggregations