Examples with FileReference org.apache.hyracks.api.io.FileReference used on opensource projects

Example 1 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class MetadataBootstrap method enlistMetadataDataset.

/**
     * Enlist a metadata index so it is available for metadata operations
     * should be performed upon bootstrapping
     *
     * @param index
     * @throws HyracksDataException
     */
public static void enlistMetadataDataset(INCServiceContext ncServiceCtx, IMetadataIndex index) throws HyracksDataException {
    ClusterPartition metadataPartition = appContext.getMetadataProperties().getMetadataPartition();
    int metadataDeviceId = metadataPartition.getIODeviceNum();
    String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath(ClusterProperties.INSTANCE.getStorageDirectoryName(), metadataPartition.getPartitionId());
    String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath();
    FileReference file = ioManager.getFileReference(metadataDeviceId, resourceName);
    index.setFile(file);
    ITypeTraits[] typeTraits = index.getTypeTraits();
    IBinaryComparatorFactory[] cmpFactories = index.getKeyBinaryComparatorFactory();
    int[] bloomFilterKeyFields = index.getBloomFilterKeyFields();
    // opTrackerProvider and ioOpCallbackFactory should both be acquired through IStorageManager
    // We are unable to do this since IStorageManager needs a dataset to determine the appropriate
    // objects
    ILSMOperationTrackerFactory opTrackerFactory = index.isPrimaryIndex() ? new PrimaryIndexOperationTrackerFactory(index.getDatasetId().getId()) : new SecondaryIndexOperationTrackerFactory(index.getDatasetId().getId());
    ILSMIOOperationCallbackFactory ioOpCallbackFactory = LSMBTreeIOOperationCallbackFactory.INSTANCE;
    IStorageComponentProvider storageComponentProvider = appContext.getStorageComponentProvider();
    if (isNewUniverse()) {
        LSMBTreeLocalResourceFactory lsmBtreeFactory = new LSMBTreeLocalResourceFactory(storageComponentProvider.getStorageManager(), typeTraits, cmpFactories, null, null, null, opTrackerFactory, ioOpCallbackFactory, storageComponentProvider.getMetadataPageManagerFactory(), new AsterixVirtualBufferCacheProvider(index.getDatasetId().getId()), storageComponentProvider.getIoOperationSchedulerProvider(), appContext.getMetadataMergePolicyFactory(), GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, true, bloomFilterKeyFields, appContext.getBloomFilterFalsePositiveRate(), true, null);
        DatasetLocalResourceFactory dsLocalResourceFactory = new DatasetLocalResourceFactory(index.getDatasetId().getId(), lsmBtreeFactory);
        // TODO(amoudi) Creating the index should be done through the same code path as other indexes
        // This is to be done by having a metadata dataset associated with each index
        IIndexBuilder indexBuilder = new IndexBuilder(ncServiceCtx, storageComponentProvider.getStorageManager(), index::getResourceId, file, dsLocalResourceFactory, true);
        indexBuilder.build();
    } else {
        final LocalResource resource = localResourceRepository.get(file.getRelativePath());
        if (resource == null) {
            throw new HyracksDataException("Could not find required metadata indexes. Please delete " + appContext.getMetadataProperties().getTransactionLogDirs().get(appContext.getTransactionSubsystem().getId()) + " to intialize as a new instance. (WARNING: all data will be lost.)");
        }
        // Why do we care about metadata dataset's resource ids? why not assign them ids similar to other resources?
        if (index.getResourceId() != resource.getId()) {
            throw new HyracksDataException("Resource Id doesn't match expected metadata index resource id");
        }
        IndexDataflowHelper indexHelper = new IndexDataflowHelper(ncServiceCtx, storageComponentProvider.getStorageManager(), file);
        // Opening the index through the helper will ensure it gets instantiated
        indexHelper.open();
        indexHelper.close();
    }
}

Example 2 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class LSMBTreeWithBuddyFileManager method getTransactionFileReferenceForCommit.

@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
    FilenameFilter transactionFilter;
    File dir = new File(baseDir);
    String[] files = dir.list(transactionFileNameFilter);
    if (files.length == 0) {
        return null;
    }
    if (files.length != 1) {
        throw new HyracksDataException("More than one transaction lock found:" + files.length);
    } else {
        transactionFilter = getTransactionFileFilter(true);
        String txnFileName = dir.getPath() + File.separator + files[0];
        // get the actual transaction files
        files = dir.list(transactionFilter);
        if (files.length < 3) {
            throw new HyracksDataException("LSM Btree with buddy transaction has less than 3 files :" + files.length);
        }
        try {
            Files.delete(Paths.get(txnFileName));
        } catch (IOException e) {
            throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName);
        }
    }
    File bTreeFile = null;
    File buddyBTreeFile = null;
    File bloomFilterFile = null;
    for (String fileName : files) {
        if (fileName.endsWith(BTREE_STRING)) {
            bTreeFile = new File(dir.getPath() + File.separator + fileName);
        } else if (fileName.endsWith(BUDDY_BTREE_STRING)) {
            buddyBTreeFile = new File(dir.getPath() + File.separator + fileName);
        } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
            bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
        } else {
            throw new HyracksDataException("unrecognized file found = " + fileName);
        }
    }
    FileReference bTreeFileRef = ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath());
    FileReference buddyBTreeFileRef = ioManager.resolveAbsolutePath(buddyBTreeFile.getAbsolutePath());
    FileReference bloomFilterFileRef = ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath());
    return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef, bloomFilterFileRef);
}

Example 3 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class AbstractLSMIndexFileManager method cleanupAndGetValidFilesInternal.

protected void cleanupAndGetValidFilesInternal(FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<ComparableFileName> allFiles) throws HyracksDataException {
    String[] files = listDirFiles(baseDir, filter);
    File dir = new File(baseDir);
    for (String fileName : files) {
        FileReference fileRef = ioManager.resolveAbsolutePath(dir.getPath() + File.separator + fileName);
        if (treeFactory == null) {
            allFiles.add(new ComparableFileName(fileRef));
            continue;
        }
        TreeIndexState idxState = isValidTreeIndex(treeFactory.createIndexInstance(fileRef));
        if (idxState == TreeIndexState.VALID) {
            allFiles.add(new ComparableFileName(fileRef));
        } else if (idxState == TreeIndexState.INVALID) {
            fileRef.delete();
        }
    }
}

Example 4 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class PushRuntimeTest method createFile.

public FileSplit createFile(NodeControllerService ncs) throws IOException {
    String fileName = "f" + aInteger.getAndIncrement() + ".tmp";
    FileReference fileRef = ncs.getIoManager().getFileReference(0, fileName);
    FileUtils.deleteQuietly(fileRef.getFile());
    fileRef.getFile().createNewFile();
    return new ManagedFileSplit(ncs.getId(), fileName);
}

Example 5 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class PersistentLocalResourceRepository method delete.

@Override
public synchronized void delete(String relativePath) throws HyracksDataException {
    FileReference resourceFile = getLocalResourceFileByName(ioManager, relativePath);
    if (resourceFile.getFile().exists()) {
        resourceFile.delete();
        resourceCache.invalidate(relativePath);
        //if replication enabled, delete resource from remote replicas
        if (isReplicationEnabled && !resourceFile.getFile().getName().startsWith(STORAGE_METADATA_FILE_NAME_PREFIX)) {
            createReplicationJob(ReplicationOperation.DELETE, resourceFile);
        }
    } else {
        throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.RESOURCE_DOES_NOT_EXIST, relativePath);
    }
}