Examples with ExternalFile org.apache.asterix.external.indexing.ExternalFile used on opensource projects

Example 16 with ExternalFile

use of org.apache.asterix.external.indexing.ExternalFile in project asterixdb by apache.

the class SecondaryIndexOperationsHelper method createExternalIndexBulkModifyOp.

protected ExternalIndexBulkModifyOperatorDescriptor createExternalIndexBulkModifyOp(JobSpecification spec, int[] fieldPermutation, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor) throws AlgebricksException {
    // create a list of file ids
    int numOfDeletedFiles = 0;
    for (ExternalFile file : externalFiles) {
        if (file.getPendingOp() == ExternalFilePendingOp.DROP_OP) {
            numOfDeletedFiles++;
        }
    }
    int[] deletedFiles = new int[numOfDeletedFiles];
    int i = 0;
    for (ExternalFile file : externalFiles) {
        if (file.getPendingOp() == ExternalFilePendingOp.DROP_OP) {
            deletedFiles[i] = file.getFileNumber();
        }
    }
    ExternalIndexBulkModifyOperatorDescriptor treeIndexBulkLoadOp = new ExternalIndexBulkModifyOperatorDescriptor(spec, dataflowHelperFactory, deletedFiles, fieldPermutation, fillFactor, false, numElementsHint);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp, secondaryPartitionConstraint);
    return treeIndexBulkLoadOp;
}

Example 17 with ExternalFile

use of org.apache.asterix.external.indexing.ExternalFile in project asterixdb by apache.

the class ExternalIndexingOperations method buildFilesIndexUpdateOp.

public static JobSpecification buildFilesIndexUpdateOp(Dataset ds, List<ExternalFile> metadataFiles, List<ExternalFile> addedFiles, List<ExternalFile> appendedFiles, MetadataProvider metadataProvider) throws AlgebricksException {
    ArrayList<ExternalFile> files = new ArrayList<>();
    for (ExternalFile file : metadataFiles) {
        if (file.getPendingOp() == ExternalFilePendingOp.DROP_OP) {
            files.add(file);
        } else if (file.getPendingOp() == ExternalFilePendingOp.APPEND_OP) {
            for (ExternalFile appendedFile : appendedFiles) {
                if (appendedFile.getFileName().equals(file.getFileName())) {
                    files.add(new ExternalFile(file.getDataverseName(), file.getDatasetName(), file.getFileNumber(), file.getFileName(), file.getLastModefiedTime(), appendedFile.getSize(), ExternalFilePendingOp.NO_OP));
                }
            }
        }
    }
    for (ExternalFile file : addedFiles) {
        files.add(file);
    }
    Collections.sort(files);
    return buildFilesIndexUpdateJobSpec(ds, files, metadataProvider);
}