Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 56 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonSchemaReaderTest method testReadSchemaFromIndexFile.

@Test
public void testReadSchemaFromIndexFile() {
    try {
        CarbonFile[] carbonFiles = FileFactory.getCarbonFile(path).listFiles(new CarbonFileFilter() {

            @Override
            public boolean accept(CarbonFile file) {
                if (file == null) {
                    return false;
                }
                return file.getName().endsWith(".carbonindex");
            }
        });
        if (carbonFiles == null || carbonFiles.length < 1) {
            throw new RuntimeException("Carbon index file not exists.");
        }
        String indexFilePath = carbonFiles[0].getAbsolutePath();
        Schema schema = CarbonSchemaReader.readSchema(indexFilePath).asOriginOrder();
        assertEquals(schema.getFieldsLength(), 12);
        checkSchema(schema);
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail();
    }
}

Example 57 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class BlockletIndexFactory method getTableBlockIndexUniqueIdentifier.

private List<TableBlockIndexUniqueIdentifierWrapper> getTableBlockIndexUniqueIdentifier(IndexInputSplit distributable) throws IOException {
    List<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper = new ArrayList<>();
    SegmentBlockIndexInfo segmentBlockIndexInfo = segmentMap.get(distributable.getSegment().getSegmentNo());
    Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = null;
    if (null != segmentBlockIndexInfo) {
        tableBlockIndexUniqueIdentifiers = segmentBlockIndexInfo.getTableBlockIndexUniqueIdentifiers();
    }
    if (tableBlockIndexUniqueIdentifiers == null) {
        tableBlockIndexUniqueIdentifiers = new HashSet<>();
        Set<String> indexFiles = distributable.getSegment().getCommittedIndexFile().keySet();
        for (String indexFile : indexFiles) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(indexFile);
            String indexFileName;
            String mergeIndexName;
            if (indexFile.endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
                indexFileName = carbonFile.getName();
                mergeIndexName = null;
            } else {
                indexFileName = carbonFile.getName();
                mergeIndexName = carbonFile.getName();
            }
            String parentPath = carbonFile.getParentFile().getAbsolutePath();
            TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier = new TableBlockIndexUniqueIdentifier(parentPath, indexFileName, mergeIndexName, distributable.getSegment().getSegmentNo());
            identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
            tableBlockIndexUniqueIdentifiers.add(tableBlockIndexUniqueIdentifier);
        }
        segmentMap.put(distributable.getSegment().getSegmentNo(), new SegmentBlockIndexInfo(tableBlockIndexUniqueIdentifiers, distributable.getSegment().getSegmentMetaDataInfo()));
    } else {
        for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : tableBlockIndexUniqueIdentifiers) {
            identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, getCarbonTable()));
        }
    }
    return identifiersWrapper;
}

Example 58 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentFileStore method writeSegmentFile.

/**
 * Method to create and write the segment file, removes the temporary directories from all the
 * respective partition directories. This method is invoked only when {@link
 * CarbonCommonConstants#CARBON_MERGE_INDEX_IN_SEGMENT} is disabled.
 * @param tablePath Table path
 * @param segmentId Segment id
 * @param timeStamp FactTimeStamp
 * @param partitionNames Partition names list
 * @param indexFileNames Index files map with partition as key and index file names set as value
 * @throws IOException
 */
public static void writeSegmentFile(String tablePath, String segmentId, String timeStamp, List<String> partitionNames, Map<String, Set<String>> indexFileNames) throws IOException {
    SegmentFileStore.SegmentFile finalSegmentFile = null;
    boolean isRelativePath;
    String partitionLoc;
    for (String partition : partitionNames) {
        isRelativePath = false;
        partitionLoc = partition;
        if (partitionLoc.startsWith(tablePath)) {
            partitionLoc = partitionLoc.substring(tablePath.length());
            isRelativePath = true;
        }
        SegmentFileStore.SegmentFile segmentFile = new SegmentFileStore.SegmentFile();
        SegmentFileStore.FolderDetails folderDetails = new SegmentFileStore.FolderDetails();
        folderDetails.setFiles(indexFileNames.get(partition));
        folderDetails.setPartitions(Collections.singletonList(partitionLoc.substring(partitionLoc.indexOf("/") + 1)));
        folderDetails.setRelative(isRelativePath);
        folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage());
        segmentFile.getLocationMap().put(partitionLoc, folderDetails);
        if (finalSegmentFile != null) {
            finalSegmentFile = finalSegmentFile.merge(segmentFile);
        } else {
            finalSegmentFile = segmentFile;
        }
    }
    Objects.requireNonNull(finalSegmentFile);
    String segmentFilesLocation = CarbonTablePath.getSegmentFilesLocation(tablePath);
    CarbonFile locationFile = FileFactory.getCarbonFile(segmentFilesLocation);
    if (!locationFile.exists()) {
        locationFile.mkdirs();
    }
    String segmentFileName = SegmentFileStore.genSegmentFileName(segmentId, timeStamp);
    SegmentFileStore.writeSegmentFile(finalSegmentFile, segmentFilesLocation + "/" + segmentFileName + CarbonTablePath.SEGMENT_EXT);
    SegmentFileStore.moveFromTempFolder(finalSegmentFile, segmentId + "_" + timeStamp + ".tmp", tablePath);
}

Example 59 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentFileStore method writeSegmentFile.

/**
 * Write segment file to the metadata folder of the table.
 *
 * @param carbonTable CarbonTable
 * @param segment segment
 * @return boolean , whether write is success or fail.
 */
public static boolean writeSegmentFile(CarbonTable carbonTable, Segment segment) throws IOException {
    String tablePath = carbonTable.getTablePath();
    CarbonFile[] indexFiles = getListOfCarbonIndexFiles(segment.getSegmentPath());
    if (indexFiles != null && indexFiles.length > 0) {
        SegmentFile segmentFile = new SegmentFile();
        segmentFile.setOptions(segment.getOptions());
        FolderDetails folderDetails = new FolderDetails();
        folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage());
        folderDetails.setRelative(false);
        segmentFile.addPath(segment.getSegmentPath(), folderDetails);
        setIndexFileNamesToFolderDetails(folderDetails, indexFiles);
        String segmentFileFolder = CarbonTablePath.getSegmentFilesLocation(tablePath);
        CarbonFile carbonFile = FileFactory.getCarbonFile(segmentFileFolder);
        if (!carbonFile.exists()) {
            carbonFile.mkdirs();
        }
        // write segment info to new file.
        writeSegmentFile(segmentFile, segmentFileFolder + File.separator + segment.getSegmentFileName());
        return true;
    }
    return false;
}

Example 60 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentFileStore method moveFromTempFolder.

/**
 * Move the loaded data from source folder to destination folder.
 */
private static void moveFromTempFolder(String source, String dest) {
    CarbonFile oldFolder = FileFactory.getCarbonFile(source);
    CarbonFile[] oldFiles = oldFolder.listFiles();
    for (CarbonFile file : oldFiles) {
        file.renameForce(dest + CarbonCommonConstants.FILE_SEPARATOR + file.getName());
    }
    oldFolder.delete();
}