use of org.voltdb.VoltTable in project voltdb by VoltDB.
the class StreamSnapshotWritePlan method createSetup.
@Override
public Callable<Boolean> createSetup(String file_path, String pathType, String file_nonce, long txnId, Map<Integer, Long> partitionTransactionIds, JSONObject jsData, SystemProcedureExecutionContext context, final VoltTable result, ExtensibleSnapshotDigestData extraSnapshotData, SiteTracker tracker, HashinatorSnapshotData hashinatorData, long timestamp) {
assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());
final StreamSnapshotRequestConfig config = new StreamSnapshotRequestConfig(jsData, context.getDatabase());
final List<StreamSnapshotRequestConfig.Stream> localStreams = filterRemoteStreams(config.streams, Longs.asList(tracker.getLocalSites()));
final Set<Integer> partitionsToAdd = getPartitionsToAdd(localStreams);
/*
* The snapshot (if config.shouldTruncate) will only contain existing partitions. Write the new partition count
* down in the digest so that we can check if enough command log is collected on
* replay.
*
* When calculating the new number of partitions
* Exploit the fact that when adding partitions the highest partition id will
* always be the number of partitions - 1. This works around the issue
* where previously we were always incrementing by the number of new partitions
* which when we failed a join resulted in an inaccurate large number of partitions
* because there was no corresponding decrement when we cleaned out and then re-added
* the partitions that were being joined. Now it will increment once and stay incremented
* after the first attempt which is less than ideal because it means you are stuck trying
* to restart with the additional partitions even if you may have operated for a long time without
* them.
*
*/
final int newPartitionCount = partitionsToAdd.isEmpty() ? context.getNumberOfPartitions() : Collections.max(partitionsToAdd) + 1;
Callable<Boolean> deferredSetup = null;
// Coalesce a truncation snapshot if shouldTruncate is true
if (config.shouldTruncate) {
deferredSetup = coalesceTruncationSnapshotPlan(file_path, pathType, file_nonce, txnId, partitionTransactionIds, context, result, extraSnapshotData, tracker, hashinatorData, timestamp, newPartitionCount);
}
// Create post snapshot update hashinator work
List<Integer> localPartitions = tracker.getPartitionsForHost(context.getHostId());
if (!partitionsToAdd.isEmpty()) {
createUpdatePartitionCountTasksForSites(localPartitions, newPartitionCount);
}
// Mark snapshot start in registry
final AtomicInteger numTables = new AtomicInteger(config.tables.length);
m_snapshotRecord = SnapshotRegistry.startSnapshot(txnId, context.getHostId(), file_path, file_nonce, SnapshotFormat.STREAM, config.tables);
// table schemas for all the tables we'll snapshot on this partition
Map<Integer, byte[]> schemas = new HashMap<Integer, byte[]>();
for (final Table table : config.tables) {
VoltTable schemaTable;
if (DrRoleType.XDCR.value().equals(context.getCluster().getDrrole()) && table.getIsdred()) {
schemaTable = CatalogUtil.getVoltTable(table, CatalogUtil.DR_HIDDEN_COLUMN_INFO);
} else {
schemaTable = CatalogUtil.getVoltTable(table);
}
schemas.put(table.getRelativeIndex(), PrivateVoltTableFactory.getSchemaBytes(schemaTable));
}
List<DataTargetInfo> sdts = createDataTargets(localStreams, hashinatorData, schemas);
// For each table, create tasks where each task has a data target.
for (final Table table : config.tables) {
createTasksForTable(table, sdts, numTables, m_snapshotRecord);
result.addRow(context.getHostId(), CoreUtils.getHostnameOrAddress(), table.getTypeName(), "SUCCESS", "");
}
return deferredSetup;
}
use of org.voltdb.VoltTable in project voltdb by VoltDB.
the class CatalogUtil method getVoltTable.
/**
*
* @param catalogTable a catalog table providing the schema
* @return An empty table with the same schema as a given catalog table.
*/
public static VoltTable getVoltTable(Table catalogTable) {
List<Column> catalogColumns = CatalogUtil.getSortedCatalogItems(catalogTable.getColumns(), "index");
VoltTable.ColumnInfo[] columns = new VoltTable.ColumnInfo[catalogColumns.size()];
int i = 0;
for (Column catCol : catalogColumns) {
columns[i++] = new VoltTable.ColumnInfo(catCol.getTypeName(), VoltType.get((byte) catCol.getType()));
}
return new VoltTable(columns);
}
use of org.voltdb.VoltTable in project voltdb by VoltDB.
the class JDBCVoltVerifier method processRows.
/**
* compare each column in a batch of rows, batches are processed by this query:
* select * from export_mirror_partitioned_table where rowid between ? and ? order by rowid limit ?
* @param rvr
* @param client
* @param jdbcclient
* @return
*/
public static boolean processRows(ReadVoltRows rvr, Client client, Connection jdbcclient) {
int batchSize = 200;
long rowid = 0;
long rowCount = 0;
VoltTable v = null;
boolean checkStatus = true;
do {
try {
v = rvr.readSomeRows(rowid, batchSize);
} catch (IOException | ProcCallException e) {
e.printStackTrace();
}
rowCount = v.getRowCount();
rowid += batchSize;
rowCheckTotal += rowCount;
if (rowCount > 0) {
checkStatus = rvr.checkTable(v, jdbcclient);
// Fail fast
if (!checkStatus && FAILFAST) {
break;
}
}
System.out.println("Current row id: " + rowid);
} while (rowCount > 0);
return checkStatus;
}
use of org.voltdb.VoltTable in project voltdb by VoltDB.
the class InsertExport2 method run.
public long run(long key, long value) {
//@SuppressWarnings("deprecation")
//long key = getVoltPrivateRealTransactionIdDontUseMe();
// Critical for proper determinism: get a cluster-wide consistent Random instance
Random rand = new Random(value);
// Insert a new record
SampleRecord record = new SampleRecord(key, rand);
voltQueueSQL(exportInsert, key, value, record.rowid_group, record.type_null_tinyint, record.type_not_null_tinyint, record.type_null_smallint, record.type_not_null_smallint, record.type_null_integer, record.type_not_null_integer, record.type_null_bigint, record.type_not_null_bigint, record.type_null_timestamp, record.type_not_null_timestamp, record.type_null_float, record.type_not_null_float, record.type_null_decimal, record.type_not_null_decimal, record.type_null_varchar25, record.type_not_null_varchar25, record.type_null_varchar128, record.type_not_null_varchar128, record.type_null_varchar1024, record.type_not_null_varchar1024);
voltQueueSQL(mirrorInsert, key, value, record.rowid_group, record.type_null_tinyint, record.type_not_null_tinyint, record.type_null_smallint, record.type_not_null_smallint, record.type_null_integer, record.type_not_null_integer, record.type_null_bigint, record.type_not_null_bigint, record.type_null_timestamp, record.type_not_null_timestamp, record.type_null_float, record.type_not_null_float, record.type_null_decimal, record.type_not_null_decimal, record.type_null_varchar25, record.type_not_null_varchar25, record.type_null_varchar128, record.type_not_null_varchar128, record.type_null_varchar1024, record.type_not_null_varchar1024);
voltQueueSQL(selectCounts);
VoltTable[] result = voltExecuteSQL();
VoltTable data = result[2];
long nrows = data.getRowCount();
if (nrows > 0) {
long ck = data.fetchRow(0).getLong(0);
voltQueueSQL(updateCounts, 1l, ck);
voltExecuteSQL(true);
} else {
voltQueueSQL(insertCounts, key, 1l);
voltExecuteSQL(true);
}
return 0;
}
use of org.voltdb.VoltTable in project voltdb by VoltDB.
the class InsertImport method run.
public long run(long key, long value) {
voltQueueSQL(deleteMirrorRow, EXPECT_SCALAR_LONG, key, value);
long deletedCount = voltExecuteSQL()[0].asScalarLong();
if (deletedCount == 0) {
voltQueueSQL(importInsert, key, value);
voltExecuteSQL(true);
} else {
voltQueueSQL(selectCounts);
VoltTable[] result = voltExecuteSQL();
VoltTable data = result[0];
long nrows = data.getRowCount();
if (nrows > 0) {
long ck = data.fetchRow(0).getLong(0);
voltQueueSQL(updateCounts, deletedCount, ck);
voltExecuteSQL(true);
} else {
voltQueueSQL(insertCounts, key, deletedCount);
voltExecuteSQL(true);
}
}
return 0;
}
Aggregations