Examples with NamespaceProperties com.ms.silverking.cloud.dht.common.NamespaceProperties used on opensource projects

Example 1 with NamespaceProperties

use of com.ms.silverking.cloud.dht.common.NamespaceProperties in project SilverKing by Morgan-Stanley.

the class ClientNamespace method clone.

@Override
public Namespace clone(String childName, long minVersion) throws NamespaceCreationException {
    long creationTime;
    creationTime = SystemTimeUtil.systemTimeSource.absTimeNanos();
    switch(nsOptions.getVersionMode()) {
        case CLIENT_SPECIFIED:
            break;
        case SEQUENTIAL:
            throw new RuntimeException("Namespace.clone() not supported for version mode of SEQUENTIAL");
        case SINGLE_VERSION:
        case SYSTEM_TIME_MILLIS:
        case SYSTEM_TIME_NANOS:
            throw new RuntimeException("Namespace.clone() can only be supplied a version " + "for a version mode of CLIENT_SPECIFIED");
        default:
            throw new RuntimeException("Panic");
    }
    return session.createNamespace(childName, new NamespaceProperties(nsOptions, this.name, minVersion, creationTime));
}

Example 2 with NamespaceProperties

use of com.ms.silverking.cloud.dht.common.NamespaceProperties in project SilverKing by Morgan-Stanley.

the class ClientNamespace method clone.

@Override
public Namespace clone(String childName) throws NamespaceCreationException {
    long creationTime;
    long minVersion;
    creationTime = SystemTimeUtil.systemTimeSource.absTimeNanos();
    switch(nsOptions.getVersionMode()) {
        case SINGLE_VERSION:
            minVersion = 0;
            break;
        case CLIENT_SPECIFIED:
            throw new RuntimeException("Namespace.clone() must be provided a version " + "for version mode of CLIENT_SPECIFIED");
        case SEQUENTIAL:
            throw new RuntimeException("Namespace.clone() not supported for version mode of SEQUENTIAL");
        case SYSTEM_TIME_MILLIS:
            minVersion = SystemTimeUtil.systemTimeSource.absTimeMillis();
            break;
        case SYSTEM_TIME_NANOS:
            minVersion = SystemTimeUtil.systemTimeSource.absTimeNanos();
            break;
        default:
            throw new RuntimeException("Panic");
    }
    return session.createNamespace(childName, new NamespaceProperties(nsOptions, this.name, minVersion, creationTime));
}

Example 3 with NamespaceProperties

use of com.ms.silverking.cloud.dht.common.NamespaceProperties in project SilverKing by Morgan-Stanley.

the class NamespaceStore method recoverExisting.

static NamespaceStore recoverExisting(long ns, File nsDir, NamespaceStore parent, StoragePolicy storagePolicy, MessageGroupBase mgBase, NodeRingMaster2 ringMaster, ConcurrentMap<UUIDBase, ActiveProxyRetrieval> activeRetrievals, ZooKeeperExtended zk, String nsLinkBasePath, LinkCreationListener linkCreationListener) {
    NamespaceStore nsStore;
    NamespaceProperties nsProperties;
    int numSegmentsToPreread;
    int numSegmentsToSkipPreread;
    int segmentsPrereadSkipped;
    if (ns == NamespaceUtil.metaInfoNamespace.contextAsLong()) {
        nsProperties = NamespaceUtil.metaInfoNamespaceProperties;
    } else {
        try {
            nsProperties = NamespacePropertiesIO.read(nsDir);
        } catch (IOException ioe) {
            throw new RuntimeException(ioe);
        }
    }
    nsStore = new NamespaceStore(ns, nsDir, NamespaceStore.DirCreationMode.DoNotCreateNSDir, nsProperties, parent, mgBase, ringMaster, true, activeRetrievals);
    if (nsProperties.getOptions().getStorageType() != StorageType.RAM) {
        List<Integer> segmentNumbers;
        segmentNumbers = FileUtil.numericFilesInDirAsSortedIntegerList(nsDir);
        if (segmentNumbers.size() > 0) {
            FileSegmentRecoverer fsr;
            int headSegmentNumber;
            int headSegmentNumberIndex;
            fsr = new FileSegmentRecoverer(nsDir);
            headSegmentNumberIndex = segmentNumbers.size() - 1;
            headSegmentNumber = segmentNumbers.get(headSegmentNumberIndex);
            segmentNumbers.remove(headSegmentNumberIndex);
            numSegmentsToPreread = (int) ((long) nsPrereadGB * (1024L * 1024L * 1024L) / (long) (nsProperties.getOptions().getSegmentSize()));
            numSegmentsToSkipPreread = segmentNumbers.size() - numSegmentsToPreread;
            segmentsPrereadSkipped = 0;
            Log.warningf("segmentsToPreread: %d", numSegmentsToPreread);
            for (int i : segmentNumbers) {
                FileSegment segment;
                SegmentPrereadMode segmentPrereadMode;
                segmentPrereadMode = segmentsPrereadSkipped < numSegmentsToSkipPreread ? SegmentPrereadMode.NoPreread : SegmentPrereadMode.Preread;
                segment = null;
                try {
                    segment = fsr.recoverFullSegment(i, nsStore, segmentIndexLocation, segmentPrereadMode);
                } catch (Exception e) {
                    Log.logErrorWarning(e, "Error recovering full segment " + i);
                    Log.warningf("Attempting partial segment recovery of segment: %d %s", i, segmentPrereadMode);
                    segment = fsr.readPartialSegment(i, false);
                    Log.warning("Successfully read segment as partial segment: " + i);
                    Log.warning("Persisting segment: " + i);
                    try {
                        segment.persist();
                    } catch (IOException ioe) {
                        throw new RuntimeException("Segment persists failed for recovered segment" + i, ioe);
                    }
                    Log.warning("Persisted segment: " + i);
                    Log.warning("Resuming full segment recovery: " + i);
                    segment = fsr.recoverFullSegment(i, nsStore, segmentIndexLocation, segmentPrereadMode);
                }
                if (segmentPrereadMode == SegmentPrereadMode.Preread && segment != null) {
                    nsStore.addRecentFileSegment(i, segment);
                }
                segmentsPrereadSkipped++;
            }
            // Check for partial recovery on last
            // FUTURE - Check for corruption
            // FUTURE - If full, then do full recovery?
            nsStore.setHeadSegment(fsr.recoverPartialSegment(headSegmentNumber, nsStore));
        } else {
            nsStore.createInitialHeadSegment();
        }
    } else {
        nsStore.initRAMSegments();
    }
    return nsStore;
}

Example 4 with NamespaceProperties

use of com.ms.silverking.cloud.dht.common.NamespaceProperties in project SilverKing by Morgan-Stanley.

the class FileSegmentRecoverer method readFullSegment.

void readFullSegment(int segmentNumber, SegmentIndexLocation segmentIndexLocation, SegmentPrereadMode segmentPrereadMode) {
    Stopwatch sw;
    Log.warning("Reading full segment: ", segmentNumber);
    sw = new SimpleStopwatch();
    try {
        FileSegment segment;
        NamespaceProperties nsProperties;
        NamespaceOptions nsOptions;
        // DataSegmentWalker       dsWalker;
        nsProperties = NamespacePropertiesIO.read(nsDir);
        nsOptions = nsProperties.getOptions();
        segment = FileSegment.openReadOnly(nsDir, segmentNumber, nsOptions.getSegmentSize(), nsOptions, segmentIndexLocation, segmentPrereadMode);
        for (DHTKeyIntEntry entry : segment.getPKC()) {
            int offset;
            offset = entry.getValue();
            if (offset < 0) {
                OffsetList offsetList;
                offsetList = segment.offsetListStore.getOffsetList(-offset);
                for (int listOffset : offsetList) {
                    System.out.printf("%s\t%d\t%d\t%f\t*%d\t%d\n", entry, segment.getCreationTime(offset), segment.getVersion(listOffset), KeyUtil.keyEntropy(entry), offset, offsetList);
                // nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, segment.getVersion(listOffset));
                }
            } else {
                System.out.printf("%s\t%d\t%d\t%f\t%d\n", entry, segment.getCreationTime(offset), segment.getVersion(offset), KeyUtil.keyEntropy(entry), offset);
            // nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, segment.getVersion(offset));
            }
        }
        // dsWalker = new DataSegmentWalker(segment.dataBuf);
        // for (DataSegmentWalkEntry entry : dsWalker) {
        // nsStore.addToSizeStats(entry.getUncompressedLength(), entry.getCompressedLength());
        // }
        segment.close();
        sw.stop();
        Log.warning("Done reading full segment: ", segmentNumber + " " + sw);
    } catch (IOException ioe) {
        Log.logErrorWarning(ioe, "Unable to recover: " + segmentNumber);
    }
}

Example 5 with NamespaceProperties

use of com.ms.silverking.cloud.dht.common.NamespaceProperties in project SilverKing by Morgan-Stanley.

the class DHTSessionImpl method getClientNamespace.

private ClientNamespace getClientNamespace(String namespace) {
    ClientNamespace clientNamespace;
    Context context;
    context = namespaceCreator.createNamespace(namespace);
    clientNamespace = clientNamespaces.get(context.contextAsLong());
    if (clientNamespace == null) {
        ClientNamespace previous;
        NamespaceProperties nsProperties;
        NamespaceOptions nsOptions;
        ClientNamespace parent;
        NamespaceLinkMeta nsLinkMeta;
        nsProperties = getNamespaceProperties(namespace);
        if (nsProperties == null) {
            throw new NamespaceNotCreatedException(namespace);
        }
        nsOptions = nsProperties.getOptions();
        if (nsProperties.getParent() != null) {
            parent = getClientNamespace(nsProperties.getParent());
        } else {
            parent = null;
        }
        if (nsOptions.getAllowLinks() && nsOptions.getVersionMode() == NamespaceVersionMode.SINGLE_VERSION) {
            nsLinkMeta = getNSLinkMeta();
        } else {
            nsLinkMeta = null;
        }
        clientNamespace = new ClientNamespace(this, namespace, nsOptions, serializationRegistry, absMillisTimeSource, server, parent, nsLinkMeta);
        previous = clientNamespaces.putIfAbsent(context.contextAsLong(), clientNamespace);
        if (previous != null) {
            clientNamespace = previous;
        } else {
            if (Log.levelMet(Level.INFO)) {
                Log.info("Created client namespace: " + namespace + " " + context);
            }
            clientNamespaceList.add(clientNamespace);
        }
    }
    return clientNamespace;
}