Examples with GridEncryptionManager org.apache.ignite.internal.managers.encryption.GridEncryptionManager used on opensource projects

Example 1 with GridEncryptionManager

use of org.apache.ignite.internal.managers.encryption.GridEncryptionManager in project ignite by apache.

the class GridCacheOffheapManager method saveStoreMetadata.

/**
 * @param store Store to save metadata.
 * @throws IgniteCheckedException If failed.
 */
private void saveStoreMetadata(CacheDataStore store, Context ctx, boolean beforeDestroy, boolean needSnapshot) throws IgniteCheckedException {
    RowStore rowStore0 = store.rowStore();
    if (rowStore0 != null && (partitionStatesRestored || grp.isLocal())) {
        ((CacheFreeList) rowStore0.freeList()).saveMetadata(grp.statisticsHolderData());
        PartitionMetaStorage<SimpleDataRow> partStore = store.partStorage();
        long updCntr = store.updateCounter();
        long size = store.fullSize();
        long rmvId = globalRemoveId().get();
        byte[] updCntrsBytes = store.partUpdateCounter().getBytes();
        PageMemoryEx pageMem = (PageMemoryEx) grp.dataRegion().pageMemory();
        IgniteWriteAheadLogManager wal = this.ctx.wal();
        GridEncryptionManager encMgr = this.ctx.kernalContext().encryption();
        if (size > 0 || updCntr > 0 || !store.partUpdateCounter().sequential() || (grp.config().isEncryptionEnabled() && encMgr.getEncryptionState(grp.groupId(), store.partId()) > 0)) {
            GridDhtPartitionState state = null;
            // localPartition will not acquire writeLock here because create=false.
            GridDhtLocalPartition part = null;
            if (!grp.isLocal()) {
                if (beforeDestroy)
                    state = GridDhtPartitionState.EVICTED;
                else {
                    part = getPartition(store);
                    if (part != null && part.state() != GridDhtPartitionState.EVICTED)
                        state = part.state();
                }
                // Do not save meta for evicted partitions on next checkpoints.
                if (state == null)
                    return;
            }
            int grpId = grp.groupId();
            long partMetaId = pageMem.partitionMetaPageId(grpId, store.partId());
            long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
            try {
                long partMetaPageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
                if (partMetaPageAddr == 0L) {
                    U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage + ", beforeDestroy=" + beforeDestroy + ", size=" + size + ", updCntr=" + updCntr + ", state=" + state + ']');
                    return;
                }
                boolean changed = false;
                try {
                    PagePartitionMetaIOV3 io = PageIO.getPageIO(partMetaPageAddr);
                    long link = io.getGapsLink(partMetaPageAddr);
                    if (updCntrsBytes == null && link != 0) {
                        partStore.removeDataRowByLink(link, grp.statisticsHolderData());
                        io.setGapsLink(partMetaPageAddr, (link = 0));
                        changed = true;
                    } else if (updCntrsBytes != null && link == 0) {
                        SimpleDataRow row = new SimpleDataRow(store.partId(), updCntrsBytes);
                        partStore.insertDataRow(row, grp.statisticsHolderData());
                        io.setGapsLink(partMetaPageAddr, (link = row.link()));
                        changed = true;
                    } else if (updCntrsBytes != null && link != 0) {
                        byte[] prev = partStore.readRow(link);
                        assert prev != null : "Read null gaps using link=" + link;
                        if (!Arrays.equals(prev, updCntrsBytes)) {
                            partStore.removeDataRowByLink(link, grp.statisticsHolderData());
                            SimpleDataRow row = new SimpleDataRow(store.partId(), updCntrsBytes);
                            partStore.insertDataRow(row, grp.statisticsHolderData());
                            io.setGapsLink(partMetaPageAddr, (link = row.link()));
                            changed = true;
                        }
                    }
                    if (changed)
                        partStore.saveMetadata(grp.statisticsHolderData());
                    changed |= io.setUpdateCounter(partMetaPageAddr, updCntr);
                    changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId);
                    changed |= io.setSize(partMetaPageAddr, size);
                    int encryptIdx = 0;
                    int encryptCnt = 0;
                    if (grp.config().isEncryptionEnabled()) {
                        long reencryptState = encMgr.getEncryptionState(grpId, store.partId());
                        if (reencryptState != 0) {
                            encryptIdx = ReencryptStateUtils.pageIndex(reencryptState);
                            encryptCnt = ReencryptStateUtils.pageCount(reencryptState);
                            if (encryptIdx == encryptCnt) {
                                encMgr.setEncryptionState(grp, store.partId(), 0, 0);
                                encryptIdx = encryptCnt = 0;
                            }
                            changed |= io.setEncryptedPageIndex(partMetaPageAddr, encryptIdx);
                            changed |= io.setEncryptedPageCount(partMetaPageAddr, encryptCnt);
                        }
                    }
                    if (state != null)
                        changed |= io.setPartitionState(partMetaPageAddr, (byte) state.ordinal());
                    else
                        assert grp.isLocal() : grp.cacheOrGroupName();
                    long cntrsPageId;
                    if (grp.sharedGroup()) {
                        long initCntrPageId = io.getCountersPageId(partMetaPageAddr);
                        Map<Integer, Long> newSizes = store.cacheSizes();
                        Map<Integer, Long> prevSizes = readSharedGroupCacheSizes(pageMem, grpId, initCntrPageId);
                        if (prevSizes != null && prevSizes.equals(newSizes))
                            // Preventing modification of sizes pages for store
                            cntrsPageId = initCntrPageId;
                        else {
                            cntrsPageId = writeSharedGroupCacheSizes(pageMem, grpId, initCntrPageId, store.partId(), newSizes);
                            if (initCntrPageId == 0 && cntrsPageId != 0) {
                                io.setCountersPageId(partMetaPageAddr, cntrsPageId);
                                changed = true;
                            }
                        }
                    } else
                        cntrsPageId = 0L;
                    int pageCnt;
                    if (needSnapshot) {
                        pageCnt = this.ctx.pageStore().pages(grpId, store.partId());
                        io.setCandidatePageCount(partMetaPageAddr, size == 0 ? 0 : pageCnt);
                        if (state == OWNING) {
                            assert part != null;
                            if (!addPartition(part, ctx.partitionStatMap(), partMetaPageAddr, io, grpId, store.partId(), this.ctx.pageStore().pages(grpId, store.partId()), store.fullSize()))
                                U.warn(log, "Partition was concurrently evicted grpId=" + grpId + ", partitionId=" + part.id());
                        } else if (state == MOVING || state == RENTING) {
                            if (ctx.partitionStatMap().forceSkipIndexPartition(grpId)) {
                                if (log.isInfoEnabled())
                                    log.info("Will not include SQL indexes to snapshot because there is " + "a partition not in " + OWNING + " state [grp=" + grp.cacheOrGroupName() + ", partId=" + store.partId() + ", state=" + state + ']');
                            }
                        }
                        changed = true;
                    } else
                        pageCnt = io.getCandidatePageCount(partMetaPageAddr);
                    if (changed && isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
                        wal.log(new MetaPageUpdatePartitionDataRecordV3(grpId, partMetaId, updCntr, rmvId, // TODO: Partition size may be long
                        (int) size, cntrsPageId, state == null ? -1 : (byte) state.ordinal(), pageCnt, link, encryptIdx, encryptCnt));
                    if (changed) {
                        partStore.saveMetadata(grp.statisticsHolderData());
                        io.setPartitionMetaStoreReuseListRoot(partMetaPageAddr, partStore.metaPageId());
                    }
                } finally {
                    pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
                }
            } finally {
                pageMem.releasePage(grpId, partMetaId, partMetaPage);
            }
        } else if (needSnapshot)
            tryAddEmptyPartitionToSnapshot(store, ctx);
    } else if (needSnapshot)
        tryAddEmptyPartitionToSnapshot(store, ctx);
}

Example 2 with GridEncryptionManager

use of org.apache.ignite.internal.managers.encryption.GridEncryptionManager in project ignite by apache.

the class CacheGroupKeyChangeTest method testNodeWithOlderKeyBecameCoordinator.

/**
 * @throws Exception If failed.
 */
@Test
public void testNodeWithOlderKeyBecameCoordinator() throws Exception {
    backups = 1;
    startTestGrids(true);
    IgniteEx node0 = grid(GRID_0);
    IgniteEx node1 = grid(GRID_1);
    createEncryptedCache(node0, node1, cacheName(), null);
    int grpId = CU.cacheId(cacheName());
    stopGrid(GRID_0);
    // Changing encryption key on one node.
    node1.context().encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
    checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
    stopGrid(GRID_1);
    // The node with only the old key ID has become the coordinator.
    node0 = startGrid(GRID_0);
    assertTrue(Collections.singleton(INITIAL_KEY_ID).containsAll(node0.context().encryption().groupKeyIds(grpId)));
    node1 = startGrid(GRID_1);
    node1.cluster().state(ClusterState.ACTIVE);
    // Wait until cache will be reencrypted with the old key.
    checkGroupKey(grpId, INITIAL_KEY_ID, MAX_AWAIT_MILLIS);
    GridEncryptionManager encrMgr0 = node0.context().encryption();
    GridEncryptionManager encrMgr1 = node1.context().encryption();
    // Changing the encryption key is not possible until the WAL segment,
    // encrypted (probably) with the previous key, is deleted.
    assertThrowsAnyCause(log, () -> encrMgr1.changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS), IgniteException.class, "Cache group key change was rejected. Cannot add new key identifier, it's already present.");
    long walIdx = node1.context().cache().context().wal().currentSegment();
    // Simulate WAL segment deletion.
    for (long n = 0; n <= walIdx; n++) node1.context().encryption().onWalSegmentRemoved(walIdx);
    encrMgr1.changeCacheGroupKey(Collections.singleton(cacheName())).get(MAX_AWAIT_MILLIS);
    checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
    checkEncryptedCaches(node0, node1);
    walIdx = Math.max(node0.context().cache().context().wal().currentSegment(), node1.context().cache().context().wal().currentSegment());
    // Simulate WAL segment deletion.
    for (long n = 0; n <= walIdx; n++) {
        encrMgr0.onWalSegmentRemoved(walIdx);
        encrMgr1.onWalSegmentRemoved(walIdx);
    }
    // Make sure the previous key has been removed.
    checkKeysCount(node0, grpId, 1, MAX_AWAIT_MILLIS);
    assertEquals(encrMgr1.groupKeyIds(grpId), encrMgr0.groupKeyIds(grpId));
}

Example 3 with GridEncryptionManager

use of org.apache.ignite.internal.managers.encryption.GridEncryptionManager in project ignite by apache.

the class AbstractEncryptionTest method checkKeysCount.

/**
 * @param node Ignite node.
 * @param grpId Cache group ID.
 * @param keysCnt Expected keys count.
 */
protected void checkKeysCount(IgniteEx node, int grpId, int keysCnt, long timeout) throws IgniteInterruptedCheckedException {
    GridEncryptionManager encMgr = node.context().encryption();
    waitForCondition(() -> encMgr.groupKeyIds(grpId).size() == keysCnt, timeout);
    assertEquals(keysCnt, encMgr.groupKeyIds(grpId).size());
}

Example 4 with GridEncryptionManager

use of org.apache.ignite.internal.managers.encryption.GridEncryptionManager in project ignite by apache.

the class IndexStoragePageMemoryImplTest method memory.

/**
 * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
 *      new empty page memory.
 * @return Page memory instance.
 */
@Override
protected PageMemory memory(boolean clean) throws Exception {
    long[] sizes = new long[10];
    for (int i = 0; i < sizes.length; i++) sizes[i] = 1024 * 1024;
    DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setEncryptionSpi(new NoopEncryptionSpi());
    cfg.setMetricExporterSpi(new NoopMetricExporterSpi());
    cfg.setSystemViewExporterSpi(new JmxSystemViewExporterSpi());
    cfg.setDataStorageConfiguration(new DataStorageConfiguration());
    GridTestKernalContext cctx = new GridTestKernalContext(log, cfg);
    cctx.add(new IgnitePluginProcessor(cctx, cfg, Collections.emptyList()));
    cctx.add(new GridInternalSubscriptionProcessor(cctx));
    cctx.add(new PerformanceStatisticsProcessor(cctx));
    cctx.add(new GridEncryptionManager(cctx));
    cctx.add(new GridMetricManager(cctx));
    cctx.add(new GridSystemViewManager(cctx));
    GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(cctx, null, null, null, new NoOpPageStoreManager(), new NoOpWALManager(), null, new IgniteCacheDatabaseSharedManager(), null, null, null, null, null, null, null, null, null, null, null, null, new CacheDiagnosticManager());
    IgniteOutClosure<CheckpointProgress> clo = () -> Mockito.mock(CheckpointProgressImpl.class);
    return new PageMemoryImpl(provider, sizes, sharedCtx, sharedCtx.pageStore(), PAGE_SIZE, (fullPageId, byteBuf, tag) -> {
        assert false : "No page replacement (rotation with disk) should happen during the test";
    }, new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {

        @Override
        public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
        }
    }, () -> true, new DataRegionMetricsImpl(new DataRegionConfiguration(), cctx), PageMemoryImpl.ThrottlingPolicy.DISABLED, clo);
}

Example 5 with GridEncryptionManager

use of org.apache.ignite.internal.managers.encryption.GridEncryptionManager in project ignite by apache.

the class PageMemoryImplNoLoadTest method memory.

/**
 * @return Page memory implementation.
 */
@Override
protected PageMemory memory() throws Exception {
    File memDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
    long[] sizes = new long[10];
    for (int i = 0; i < sizes.length; i++) sizes[i] = 5 * 1024 * 1024;
    DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), memDir);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setEncryptionSpi(new NoopEncryptionSpi());
    cfg.setMetricExporterSpi(new NoopMetricExporterSpi());
    cfg.setSystemViewExporterSpi(new JmxSystemViewExporterSpi());
    cfg.setDataStorageConfiguration(new DataStorageConfiguration());
    GridTestKernalContext cctx = new GridTestKernalContext(log, cfg);
    cctx.add(new IgnitePluginProcessor(cctx, cfg, Collections.emptyList()));
    cctx.add(new GridInternalSubscriptionProcessor(cctx));
    cctx.add(new PerformanceStatisticsProcessor(cctx));
    cctx.add(new GridEncryptionManager(cctx));
    cctx.add(new GridMetricManager(cctx));
    cctx.add(new GridSystemViewManager(cctx));
    GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(cctx, null, null, null, new NoOpPageStoreManager(), new NoOpWALManager(), null, new IgniteCacheDatabaseSharedManager(), null, null, null, null, null, null, null, null, null, null, null, null, null);
    IgniteOutClosure<CheckpointProgress> clo = () -> Mockito.mock(CheckpointProgressImpl.class);
    return new PageMemoryImpl(provider, sizes, sharedCtx, sharedCtx.pageStore(), PAGE_SIZE, (fullPageId, byteBuf, tag) -> {
        assert false : "No page replacement (rotation with disk) should happen during the test";
    }, new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {

        @Override
        public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
        }
    }, () -> true, new DataRegionMetricsImpl(new DataRegionConfiguration(), cctx), PageMemoryImpl.ThrottlingPolicy.DISABLED, clo);
}