Examples with ILSMMergePolicyFactory org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory used on opensource projects

Example 1 with ILSMMergePolicyFactory

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory in project asterixdb by apache.

the class QueryTranslator method validateCompactionPolicy.

protected static void validateCompactionPolicy(String compactionPolicy, Map<String, String> compactionPolicyProperties, MetadataTransactionContext mdTxnCtx, boolean isExternalDataset) throws CompilationException, Exception {
    CompactionPolicy compactionPolicyEntity = MetadataManager.INSTANCE.getCompactionPolicy(mdTxnCtx, MetadataConstants.METADATA_DATAVERSE_NAME, compactionPolicy);
    if (compactionPolicyEntity == null) {
        throw new CompilationException("Unknown compaction policy: " + compactionPolicy);
    }
    String compactionPolicyFactoryClassName = compactionPolicyEntity.getClassName();
    ILSMMergePolicyFactory mergePolicyFactory = (ILSMMergePolicyFactory) Class.forName(compactionPolicyFactoryClassName).newInstance();
    if (isExternalDataset && mergePolicyFactory.getName().compareTo("correlated-prefix") == 0) {
        throw new CompilationException("The correlated-prefix merge policy cannot be used with external dataset.");
    }
    if (compactionPolicyProperties == null) {
        if (mergePolicyFactory.getName().compareTo("no-merge") != 0) {
            throw new CompilationException("Compaction policy properties are missing.");
        }
    } else {
        for (Map.Entry<String, String> entry : compactionPolicyProperties.entrySet()) {
            if (!mergePolicyFactory.getPropertiesNames().contains(entry.getKey())) {
                throw new CompilationException("Invalid compaction policy property: " + entry.getKey());
            }
        }
        for (String p : mergePolicyFactory.getPropertiesNames()) {
            if (!compactionPolicyProperties.containsKey(p)) {
                throw new CompilationException("Missing compaction policy property: " + p);
            }
        }
    }
}

Example 2 with ILSMMergePolicyFactory

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory in project asterixdb by apache.

the class DatasetUtil method getMergePolicyFactory.

public static Pair<ILSMMergePolicyFactory, Map<String, String>> getMergePolicyFactory(Dataset dataset, MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
    String policyName = dataset.getCompactionPolicy();
    CompactionPolicy compactionPolicy = MetadataManager.INSTANCE.getCompactionPolicy(mdTxnCtx, MetadataConstants.METADATA_DATAVERSE_NAME, policyName);
    String compactionPolicyFactoryClassName = compactionPolicy.getClassName();
    ILSMMergePolicyFactory mergePolicyFactory;
    Map<String, String> properties = dataset.getCompactionPolicyProperties();
    try {
        mergePolicyFactory = (ILSMMergePolicyFactory) Class.forName(compactionPolicyFactoryClassName).newInstance();
        if (mergePolicyFactory.getName().compareTo(CorrelatedPrefixMergePolicyFactory.NAME) == 0) {
            properties.put(CorrelatedPrefixMergePolicyFactory.KEY_DATASET_ID, Integer.toString(dataset.getDatasetId()));
        }
    } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
        throw new AlgebricksException(e);
    }
    return new Pair<>(mergePolicyFactory, properties);
}

Example 3 with ILSMMergePolicyFactory

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory in project asterixdb by apache.

the class DatasetUtil method createDatasetJobSpec.

public static JobSpecification createDatasetJobSpec(Dataset dataset, MetadataProvider metadataProvider) throws AlgebricksException {
    Index index = IndexUtil.getPrimaryIndex(dataset);
    ARecordType itemType = (ARecordType) metadataProvider.findType(dataset);
    // get meta item type
    ARecordType metaItemType = null;
    if (dataset.hasMetaPart()) {
        metaItemType = (ARecordType) metadataProvider.findMetaType(dataset);
    }
    JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
    Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset);
    FileSplit[] fs = splitsAndConstraint.first.getFileSplits();
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < fs.length; i++) {
        sb.append(fs[i] + " ");
    }
    LOGGER.info("CREATING File Splits: " + sb.toString());
    Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
    //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
    IResourceFactory resourceFactory = dataset.getResourceFactory(metadataProvider, index, itemType, metaItemType, compactionInfo.first, compactionInfo.second);
    IndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(metadataProvider.getStorageComponentProvider().getStorageManager(), splitsAndConstraint.first, resourceFactory, !dataset.isTemp());
    IndexCreateOperatorDescriptor indexCreateOp = new IndexCreateOperatorDescriptor(spec, indexBuilderFactory);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexCreateOp, splitsAndConstraint.second);
    spec.addRoot(indexCreateOp);
    return spec;
}

Example 4 with ILSMMergePolicyFactory

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory in project asterixdb by apache.

the class ExternalIndexingOperations method buildFilesIndexCreateJobSpec.

public static JobSpecification buildFilesIndexCreateJobSpec(Dataset dataset, List<ExternalFile> externalFilesSnapshot, MetadataProvider metadataProvider) throws AlgebricksException {
    IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
    JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
    Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
    ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
    Map<String, String> mergePolicyProperties = compactionInfo.second;
    Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset, IndexingConstants.getFilesIndexName(dataset.getDatasetName()));
    IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
    String fileIndexName = IndexingConstants.getFilesIndexName(dataset.getDatasetName());
    Index fileIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataset.getDataverseName(), dataset.getDatasetName(), fileIndexName);
    ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
    IResourceFactory resourceFactory = dataset.getResourceFactory(metadataProvider, fileIndex, recordType, null, mergePolicyFactory, mergePolicyProperties);
    IIndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(storageComponentProvider.getStorageManager(), secondaryFileSplitProvider, resourceFactory, !dataset.isTemp());
    IIndexDataflowHelperFactory dataflowHelperFactory = new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), secondaryFileSplitProvider);
    ExternalFilesIndexCreateOperatorDescriptor externalFilesOp = new ExternalFilesIndexCreateOperatorDescriptor(spec, indexBuilderFactory, dataflowHelperFactory, externalFilesSnapshot);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, externalFilesOp, secondarySplitsAndConstraint.second);
    spec.addRoot(externalFilesOp);
    spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
    return spec;
}