Search in sources :

Example 1 with CREATE

use of java.nio.file.StandardOpenOption.CREATE in project ignite by apache.

the class FileWriteAheadLogManager method formatWorkSegments.

/**
 * Formatting working segments to {@link DataStorageConfiguration#getWalSegmentSize()} for work in a mmap or fsync case.
 *
 * @throws StorageException If an error occurs when formatting.
 */
private void formatWorkSegments() throws StorageException {
    assert isArchiverEnabled();
    if (mode == WALMode.FSYNC || mmap) {
        List<FileDescriptor> toFormat = Arrays.stream(scan(walWorkDir.listFiles(WAL_SEGMENT_FILE_FILTER))).filter(fd -> fd.file().length() < dsCfg.getWalSegmentSize()).collect(toList());
        if (!toFormat.isEmpty()) {
            if (log.isInfoEnabled()) {
                log.info("WAL segments in working directory should have the same size: '" + U.humanReadableByteCount(dsCfg.getWalSegmentSize()) + "'. Segments that need reformat " + "found: " + F.viewReadOnly(toFormat, fd -> fd.file().getName()) + '.');
            }
            for (int i = 0, j = 0; i < toFormat.size(); i++) {
                FileDescriptor fd = toFormat.get(i);
                File tmpDst = new File(fd.file().getName() + TMP_SUFFIX);
                try {
                    Files.copy(fd.file().toPath(), tmpDst.toPath());
                    if (log.isDebugEnabled()) {
                        log.debug("Start formatting WAL segment [filePath=" + tmpDst.getAbsolutePath() + ", fileSize=" + U.humanReadableByteCount(tmpDst.length()) + ", toSize=" + U.humanReadableByteCount(dsCfg.getWalSegmentSize()) + ']');
                    }
                    try (FileIO fileIO = ioFactory.create(tmpDst, CREATE, READ, WRITE)) {
                        int left = (int) (dsCfg.getWalSegmentSize() - tmpDst.length());
                        fileIO.position(tmpDst.length());
                        while (left > 0) left -= fileIO.writeFully(FILL_BUF, 0, Math.min(FILL_BUF.length, left));
                        fileIO.force();
                    }
                    Files.move(tmpDst.toPath(), fd.file().toPath(), REPLACE_EXISTING, ATOMIC_MOVE);
                    if (log.isDebugEnabled())
                        log.debug("WAL segment formatted: " + fd.file().getAbsolutePath());
                    // Batch output.
                    if (log.isInfoEnabled() && (i == toFormat.size() - 1 || (i != 0 && i % 9 == 0))) {
                        log.info("WAL segments formatted: " + toFormat.get(j).file().getName() + (i == j ? "" : " - " + fileName(i)));
                        j = i + 1;
                    }
                } catch (IOException e) {
                    throw new StorageException("Failed to format WAL segment: " + fd.file().getAbsolutePath(), e);
                }
            }
        }
    }
}
Also used : Arrays(java.util.Arrays) BufferedInputStream(java.io.BufferedInputStream) GridFutureAdapter(org.apache.ignite.internal.util.future.GridFutureAdapter) GridFinishedFuture(org.apache.ignite.internal.util.future.GridFinishedFuture) DataStorageMetricsImpl(org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl) FileIO(org.apache.ignite.internal.processors.cache.persistence.file.FileIO) Map(java.util.Map) IGNITE_WAL_SERIALIZER_VERSION(org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION) WALMode(org.apache.ignite.configuration.WALMode) SegmentIO(org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO) CIX1(org.apache.ignite.internal.util.typedef.CIX1) RecordV1Serializer.readPosition(org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readPosition) HALF_MAX_WAL_ARCHIVE_SIZE(org.apache.ignite.configuration.DataStorageConfiguration.HALF_MAX_WAL_ARCHIVE_SIZE) RandomAccessFileIOFactory(org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory) RecordSerializer(org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) IgniteDataIntegrityViolationException(org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException) IGNITE_THRESHOLD_WAIT_TIME_NEXT_WAL_SEGMENT(org.apache.ignite.IgniteSystemProperties.IGNITE_THRESHOLD_WAIT_TIME_NEXT_WAL_SEGMENT) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) Set(java.util.Set) ClosedByInterruptException(java.nio.channels.ClosedByInterruptException) ByteOrder(java.nio.ByteOrder) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) RecordSerializerFactoryImpl(org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl) EVT_WAL_SEGMENT_COMPACTED(org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_COMPACTED) SYSTEM_WORKER_TERMINATION(org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION) CompressionProcessor.getDefaultCompressionLevel(org.apache.ignite.internal.processors.compress.CompressionProcessor.getDefaultCompressionLevel) MarshalledRecord(org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord) IgniteWriteAheadLogManager(org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) ZipOutputStream(java.util.zip.ZipOutputStream) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteBiPredicate(org.apache.ignite.lang.IgniteBiPredicate) IgniteThread(org.apache.ignite.thread.IgniteThread) ZIP_SUFFIX(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.ZIP_SUFFIX) U(org.apache.ignite.internal.util.typedef.internal.U) EVT_WAL_SEGMENT_ARCHIVED(org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED) IgniteLogger(org.apache.ignite.IgniteLogger) LATEST_SERIALIZER_VERSION(org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory.LATEST_SERIALIZER_VERSION) BufferedOutputStream(java.io.BufferedOutputStream) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) READ(java.nio.file.StandardOpenOption.READ) DiskPageCompression(org.apache.ignite.configuration.DiskPageCompression) GridCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager) FileHandleManager(org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileHandleManager) CI1(org.apache.ignite.internal.util.typedef.CI1) IgniteInterruptedCheckedException(org.apache.ignite.internal.IgniteInterruptedCheckedException) CompressionProcessor.checkCompressionLevelBounds(org.apache.ignite.internal.processors.compress.CompressionProcessor.checkCompressionLevelBounds) SwitchSegmentRecord(org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord) REPLACE_EXISTING(java.nio.file.StandardCopyOption.REPLACE_EXISTING) IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT(org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT) FileDescriptor.fileName(org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor.fileName) ATOMIC_MOVE(java.nio.file.StandardCopyOption.ATOMIC_MOVE) Files(java.nio.file.Files) FileOutputStream(java.io.FileOutputStream) GridUnsafe(org.apache.ignite.internal.util.GridUnsafe) IOException(java.io.IOException) FileInput(org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput) UNLIMITED_WAL_ARCHIVE(org.apache.ignite.configuration.DataStorageConfiguration.UNLIMITED_WAL_ARCHIVE) GridFileUtils(org.apache.ignite.internal.util.io.GridFileUtils) File(java.io.File) AtomicLong(java.util.concurrent.atomic.AtomicLong) FileFilter(java.io.FileFilter) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) TreeMap(java.util.TreeMap) FileHandleManagerFactory(org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileHandleManagerFactory) CREATE(java.nio.file.StandardOpenOption.CREATE) LockedSegmentFileInputFactory(org.apache.ignite.internal.processors.cache.persistence.wal.io.LockedSegmentFileInputFactory) WalSegmentArchivedEvent(org.apache.ignite.events.WalSegmentArchivedEvent) SegmentFileInputFactory(org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory) CO(org.apache.ignite.internal.util.typedef.CO) IgniteUuid(org.apache.ignite.lang.IgniteUuid) IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) StorageException(org.apache.ignite.internal.processors.cache.persistence.StorageException) CRITICAL_ERROR(org.apache.ignite.failure.FailureType.CRITICAL_ERROR) Time(java.sql.Time) IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE(org.apache.ignite.IgniteSystemProperties.IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE) FileWriteHandle(org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileWriteHandle) DATA_RECORD_V2(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD_V2) GridWorker(org.apache.ignite.internal.util.worker.GridWorker) ByteBuffer(java.nio.ByteBuffer) PriorityBlockingQueue(java.util.concurrent.PriorityBlockingQueue) IgniteSystemProperties(org.apache.ignite.IgniteSystemProperties) IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE(org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE) X(org.apache.ignite.internal.util.typedef.X) FailureProcessor(org.apache.ignite.internal.processors.failure.FailureProcessor) ZipEntry(java.util.zip.ZipEntry) WRITE(java.nio.file.StandardOpenOption.WRITE) RecordV1Serializer(org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer) FailureType(org.apache.ignite.failure.FailureType) AtomicReferenceFieldUpdater(java.util.concurrent.atomic.AtomicReferenceFieldUpdater) Collection(java.util.Collection) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) AbstractFileHandle(org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.AbstractFileHandle) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) EOFException(java.io.EOFException) FileNotFoundException(java.io.FileNotFoundException) GridCacheSharedManagerAdapter(org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) Objects(java.util.Objects) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) FastCrc(org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc) WalSegmentCompactedEvent(org.apache.ignite.events.WalSegmentCompactedEvent) IGNITE_WAL_MMAP(org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP) DataInput(java.io.DataInput) HeaderRecord(org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord) Pattern(java.util.regex.Pattern) RecordSerializerFactory(org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory) AtomicLongArray(java.util.concurrent.atomic.AtomicLongArray) ZipInputStream(java.util.zip.ZipInputStream) WALDisableContext(org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext) PdsFolderSettings(org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings) HashMap(java.util.HashMap) GridTimeoutProcessor(org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor) IgniteSystemProperties.getDouble(org.apache.ignite.IgniteSystemProperties.getDouble) HEADER_RECORD_SIZE(org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE) HashSet(java.util.HashSet) FailureContext(org.apache.ignite.failure.FailureContext) SegmentAware(org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) TMP_SUFFIX(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.TMP_SUFFIX) FileIOFactory(org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory) DataStorageConfiguration(org.apache.ignite.configuration.DataStorageConfiguration) SimpleSegmentFileInputFactory(org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory) MemoryRecoveryRecord(org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord) RolloverType(org.apache.ignite.internal.pagemem.wal.record.RolloverType) F(org.apache.ignite.internal.util.typedef.F) RecordV1Serializer.readSegmentHeader(org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader) GridTimeoutObject(org.apache.ignite.internal.processors.timeout.GridTimeoutObject) FileInputStream(java.io.FileInputStream) FileAlreadyExistsException(java.nio.file.FileAlreadyExistsException) Collectors.toList(java.util.stream.Collectors.toList) GridEventStorageManager(org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager) Collections(java.util.Collections) IOException(java.io.IOException) File(java.io.File) StorageException(org.apache.ignite.internal.processors.cache.persistence.StorageException) FileIO(org.apache.ignite.internal.processors.cache.persistence.file.FileIO)

Aggregations

BufferedInputStream (java.io.BufferedInputStream)1 BufferedOutputStream (java.io.BufferedOutputStream)1 DataInput (java.io.DataInput)1 EOFException (java.io.EOFException)1 File (java.io.File)1 FileFilter (java.io.FileFilter)1 FileInputStream (java.io.FileInputStream)1 FileNotFoundException (java.io.FileNotFoundException)1 FileOutputStream (java.io.FileOutputStream)1 IOException (java.io.IOException)1 ByteBuffer (java.nio.ByteBuffer)1 ByteOrder (java.nio.ByteOrder)1 ClosedByInterruptException (java.nio.channels.ClosedByInterruptException)1 FileAlreadyExistsException (java.nio.file.FileAlreadyExistsException)1 Files (java.nio.file.Files)1 ATOMIC_MOVE (java.nio.file.StandardCopyOption.ATOMIC_MOVE)1 REPLACE_EXISTING (java.nio.file.StandardCopyOption.REPLACE_EXISTING)1 CREATE (java.nio.file.StandardOpenOption.CREATE)1 READ (java.nio.file.StandardOpenOption.READ)1 WRITE (java.nio.file.StandardOpenOption.WRITE)1