use of io.dingodb.store.row.util.Pair in project dingo by dingodb.
the class MemoryRawKVStore method doSnapshotSave.
void doSnapshotSave(final MemoryKVStoreSnapshotFile snapshotFile, final String snapshotPath, final Region region) throws Exception {
final Timer.Context timeCtx = getTimeContext("SNAPSHOT_SAVE");
try {
final String tempPath = snapshotPath + "_temp";
final File tempFile = new File(tempPath);
FileUtils.deleteDirectory(tempFile);
FileUtils.forceMkdir(tempFile);
snapshotFile.writeToFile(tempPath, SEQUENCE_DB, new MemoryKVStoreSnapshotFile.SequenceDB(subRangeMap(this.sequenceDB, region)));
snapshotFile.writeToFile(tempPath, FENCING_KEY_DB, new MemoryKVStoreSnapshotFile.FencingKeyDB(subRangeMap(this.fencingKeyDB, region)));
snapshotFile.writeToFile(tempPath, LOCKER_DB, new MemoryKVStoreSnapshotFile.LockerDB(subRangeMap(this.lockerDB, region)));
final int size = this.opts.getKeysPerSegment();
final List<Pair<byte[], byte[]>> segment = Lists.newArrayListWithCapacity(size);
int index = 0;
final byte[] realStartKey = BytesUtil.nullToEmpty(region.getStartKey());
final byte[] endKey = region.getEndKey();
final NavigableMap<byte[], byte[]> subMap;
if (endKey == null) {
subMap = this.defaultDB.tailMap(realStartKey);
} else {
subMap = this.defaultDB.subMap(realStartKey, endKey);
}
for (final Map.Entry<byte[], byte[]> entry : subMap.entrySet()) {
segment.add(Pair.of(entry.getKey(), entry.getValue()));
if (segment.size() >= size) {
snapshotFile.writeToFile(tempPath, SEGMENT + index++, new MemoryKVStoreSnapshotFile.Segment(segment));
segment.clear();
}
}
if (!segment.isEmpty()) {
snapshotFile.writeToFile(tempPath, SEGMENT + index++, new MemoryKVStoreSnapshotFile.Segment(segment));
segment.clear();
}
snapshotFile.writeToFile(tempPath, TAIL_INDEX, new MemoryKVStoreSnapshotFile.TailIndex(--index));
final File destinationPath = new File(snapshotPath);
FileUtils.deleteDirectory(destinationPath);
FileUtils.moveDirectory(tempFile, destinationPath);
} finally {
timeCtx.stop();
}
}
use of io.dingodb.store.row.util.Pair in project dingo by dingodb.
the class RegionHeartbeatHandler method tryToFindLaziestWorker.
private Pair<String, Integer> tryToFindLaziestWorker(final List<Pair<String, Integer>> lazyWorkers) {
final List<Pair<Pair<String, Integer>, StoreStats>> storeStatsList = Lists.newArrayList();
for (final Pair<String, Integer> worker : lazyWorkers) {
final StoreStats stats = rowStoreMetaAdaptor.storeStats(GeneralId.fromStr(worker.getKey()));
if (stats != null) {
// TODO check timeInterval
storeStatsList.add(Pair.of(worker, stats));
}
}
if (storeStatsList.isEmpty()) {
return null;
}
if (storeStatsList.size() == 1) {
return storeStatsList.get(0).getKey();
}
final Pair<Pair<String, Integer>, StoreStats> min = Collections.min(storeStatsList, (o1, o2) -> {
final StoreStats s1 = o1.getValue();
final StoreStats s2 = o2.getValue();
int val = Boolean.compare(s1.isBusy(), s2.isBusy());
if (val != 0) {
return val;
}
val = Integer.compare(s1.getRegionCount(), s2.getRegionCount());
if (val != 0) {
return val;
}
val = Long.compare(s1.getBytesWritten(), s2.getBytesWritten());
if (val != 0) {
return val;
}
val = Long.compare(s1.getBytesRead(), s2.getBytesRead());
if (val != 0) {
return val;
}
val = Long.compare(s1.getKeysWritten(), s2.getKeysWritten());
if (val != 0) {
return val;
}
val = Long.compare(s1.getKeysRead(), s2.getKeysRead());
if (val != 0) {
return val;
}
return Long.compare(-s1.getAvailable(), -s2.getAvailable());
});
return min.getKey();
}
use of io.dingodb.store.row.util.Pair in project dingo by dingodb.
the class RegionHeartbeatHandler method balance.
private Instruction balance(Region region, RegionStats regionStats) {
final long clusterId = 0;
final String storeId = regionStats.getLeader().getStoreId();
final ClusterStatsManager clusterStatsManager = ClusterStatsManager.getInstance(clusterId);
clusterStatsManager.addOrUpdateLeader(storeId, region.getId());
// check if the modelWorker
final Pair<Set<String>, Integer> modelWorkers = clusterStatsManager.findModelWorkerStores(1);
final Set<String> modelWorkerStoreIds = modelWorkers.getKey();
final int modelWorkerLeaders = modelWorkers.getValue();
if (!modelWorkerStoreIds.contains(storeId)) {
return null;
}
log.info("[Cluster] model worker stores is: {}, it has {} leaders.", modelWorkerStoreIds, modelWorkerLeaders);
final List<Peer> peers = region.getPeers();
if (peers == null) {
return null;
}
final List<Endpoint> endpoints = Lists.transform(peers, Peer::getEndpoint);
final Map<String, Endpoint> storeIds = rowStoreMetaAdaptor.storeLocation();
// find lazyWorkers
final List<Pair<String, Integer>> lazyWorkers = clusterStatsManager.findLazyWorkerStores(storeIds.keySet());
if (lazyWorkers.isEmpty()) {
return null;
}
for (int i = lazyWorkers.size() - 1; i >= 0; i--) {
final Pair<String, Integer> worker = lazyWorkers.get(i);
if (modelWorkerLeaders - worker.getValue() <= 1) {
// no need to transfer
lazyWorkers.remove(i);
}
}
if (lazyWorkers.isEmpty()) {
return null;
}
final Pair<String, Integer> laziestWorker = tryToFindLaziestWorker(lazyWorkers);
if (laziestWorker == null) {
return null;
}
final String lazyWorkerStoreId = laziestWorker.getKey();
log.info("[Cluster: {}], lazy worker store is: {}, it has {} leaders.", clusterId, lazyWorkerStoreId, laziestWorker.getValue());
final Instruction.TransferLeader transferLeader = new Instruction.TransferLeader();
transferLeader.setMoveToStoreId(lazyWorkerStoreId);
transferLeader.setMoveToEndpoint(storeIds.get(lazyWorkerStoreId));
final Instruction instruction = new Instruction();
instruction.setRegion(region.copy());
instruction.setTransferLeader(transferLeader);
log.info("[Cluster: {}], send 'instruction.transferLeader': {} to region: {}.", clusterId, instruction, region);
return instruction;
}
Aggregations