Examples with CarbonTablePath org.apache.carbondata.core.util.path.CarbonTablePath used on opensource projects

Example 26 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonLoaderUtil method isValidSegment.

/**
   * the method returns true if the segment has carbondata file else returns false.
   *
   * @param loadModel
   * @param currentLoad
   * @return
   */
public static boolean isValidSegment(CarbonLoadModel loadModel, int currentLoad) {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    int fileCount = 0;
    int partitionCount = carbonTable.getPartitionCount();
    for (int i = 0; i < partitionCount; i++) {
        String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "", currentLoad + "");
        CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
        CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

            @Override
            public boolean accept(CarbonFile file) {
                return file.getName().endsWith(CarbonTablePath.getCarbonIndexExtension()) || file.getName().endsWith(CarbonTablePath.getCarbonDataExtension());
            }
        });
        fileCount += files.length;
        if (files.length > 0) {
            return true;
        }
    }
    if (fileCount == 0) {
        return false;
    }
    return true;
}

Example 27 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonLoaderUtil method deleteSegment.

public static void deleteSegment(CarbonLoadModel loadModel, int currentLoad) {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
        String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "", currentLoad + "");
        deleteStorePath(segmentPath);
    }
}

Example 28 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonLoaderUtil method recordLoadMetadata.

/**
   * This API will write the load level metadata for the loadmanagement module inorder to
   * manage the load and query execution management smoothly.
   *
   * @param loadCount
   * @param loadMetadataDetails
   * @param loadModel
   * @param loadStatus
   * @param startLoadTime
   * @return boolean which determines whether status update is done or not.
   * @throws IOException
   */
public static boolean recordLoadMetadata(int loadCount, LoadMetadataDetails loadMetadataDetails, CarbonLoadModel loadModel, String loadStatus, long startLoadTime) throws IOException {
    boolean status = false;
    String metaDataFilepath = loadModel.getCarbonDataLoadSchema().getCarbonTable().getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableStatusPath = carbonTablePath.getTableStatusFilePath();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    try {
        if (carbonLock.lockWithRetries()) {
            LOGGER.info("Acquired lock for table" + loadModel.getDatabaseName() + "." + loadModel.getTableName() + " for table status updation");
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaDataFilepath);
            long loadEnddate = CarbonUpdateUtil.readCurrentTime();
            loadMetadataDetails.setLoadEndTime(loadEnddate);
            loadMetadataDetails.setLoadStatus(loadStatus);
            loadMetadataDetails.setLoadName(String.valueOf(loadCount));
            loadMetadataDetails.setLoadStartTime(startLoadTime);
            List<LoadMetadataDetails> listOfLoadFolderDetails = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
            if (null != listOfLoadFolderDetailsArray) {
                Collections.addAll(listOfLoadFolderDetails, listOfLoadFolderDetailsArray);
            }
            listOfLoadFolderDetails.add(loadMetadataDetails);
            SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetails.toArray(new LoadMetadataDetails[listOfLoadFolderDetails.size()]));
            status = true;
        } else {
            LOGGER.error("Not able to acquire the lock for Table status updation for table " + loadModel.getDatabaseName() + "." + loadModel.getTableName());
        }
    } finally {
        if (carbonLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + loadModel.getDatabaseName() + "." + loadModel.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + loadModel.getDatabaseName() + "." + loadModel.getTableName() + " during table status updation");
        }
    }
    return status;
}

Example 29 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class DeleteLoadFolders method getSegmentPath.

/**
   * returns segment path
   *
   * @param dbName
   * @param tableName
   * @param storeLocation
   * @param partitionId
   * @param oneLoad
   * @return
   */
private static String getSegmentPath(String dbName, String tableName, String storeLocation, int partitionId, LoadMetadataDetails oneLoad) {
    CarbonTablePath carbon = new CarbonStorePath(storeLocation).getCarbonTablePath(new CarbonTableIdentifier(dbName, tableName, ""));
    String segmentId = oneLoad.getLoadName();
    return carbon.getCarbonDataDirectoryPath("" + partitionId, segmentId);
}

Example 30 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class StoreCreator method createTable.

private static CarbonTable createTable() throws IOException {
    TableInfo tableInfo = new TableInfo();
    tableInfo.setStorePath(absoluteTableIdentifier.getStorePath());
    tableInfo.setDatabaseName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName());
    TableSchema tableSchema = new TableSchema();
    tableSchema.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
    List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
    ArrayList<Encoding> encodings = new ArrayList<>();
    encodings.add(Encoding.DICTIONARY);
    ColumnSchema id = new ColumnSchema();
    id.setColumnName("ID");
    id.setColumnar(true);
    id.setDataType(DataType.INT);
    id.setEncodingList(encodings);
    id.setColumnUniqueId(UUID.randomUUID().toString());
    id.setDimensionColumn(true);
    id.setColumnGroup(1);
    columnSchemas.add(id);
    ColumnSchema date = new ColumnSchema();
    date.setColumnName("date");
    date.setColumnar(true);
    date.setDataType(DataType.STRING);
    date.setEncodingList(encodings);
    date.setColumnUniqueId(UUID.randomUUID().toString());
    date.setDimensionColumn(true);
    date.setColumnGroup(2);
    date.setSortColumn(true);
    columnSchemas.add(date);
    ColumnSchema country = new ColumnSchema();
    country.setColumnName("country");
    country.setColumnar(true);
    country.setDataType(DataType.STRING);
    country.setEncodingList(encodings);
    country.setColumnUniqueId(UUID.randomUUID().toString());
    country.setDimensionColumn(true);
    country.setColumnGroup(3);
    country.setSortColumn(true);
    columnSchemas.add(country);
    ColumnSchema name = new ColumnSchema();
    name.setColumnName("name");
    name.setColumnar(true);
    name.setDataType(DataType.STRING);
    name.setEncodingList(encodings);
    name.setColumnUniqueId(UUID.randomUUID().toString());
    name.setDimensionColumn(true);
    name.setColumnGroup(4);
    name.setSortColumn(true);
    columnSchemas.add(name);
    ColumnSchema phonetype = new ColumnSchema();
    phonetype.setColumnName("phonetype");
    phonetype.setColumnar(true);
    phonetype.setDataType(DataType.STRING);
    phonetype.setEncodingList(encodings);
    phonetype.setColumnUniqueId(UUID.randomUUID().toString());
    phonetype.setDimensionColumn(true);
    phonetype.setColumnGroup(5);
    phonetype.setSortColumn(true);
    columnSchemas.add(phonetype);
    ColumnSchema serialname = new ColumnSchema();
    serialname.setColumnName("serialname");
    serialname.setColumnar(true);
    serialname.setDataType(DataType.STRING);
    serialname.setEncodingList(encodings);
    serialname.setColumnUniqueId(UUID.randomUUID().toString());
    serialname.setDimensionColumn(true);
    serialname.setColumnGroup(6);
    serialname.setSortColumn(true);
    columnSchemas.add(serialname);
    ColumnSchema salary = new ColumnSchema();
    salary.setColumnName("salary");
    salary.setColumnar(true);
    salary.setDataType(DataType.INT);
    salary.setEncodingList(new ArrayList<Encoding>());
    salary.setColumnUniqueId(UUID.randomUUID().toString());
    salary.setDimensionColumn(false);
    salary.setColumnGroup(7);
    columnSchemas.add(salary);
    tableSchema.setListOfColumns(columnSchemas);
    SchemaEvolution schemaEvol = new SchemaEvolution();
    schemaEvol.setSchemaEvolutionEntryList(new ArrayList<SchemaEvolutionEntry>());
    tableSchema.setSchemaEvalution(schemaEvol);
    tableSchema.setTableId(UUID.randomUUID().toString());
    tableInfo.setTableUniqueName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName() + "_" + absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
    tableInfo.setLastUpdatedTime(System.currentTimeMillis());
    tableInfo.setFactTable(tableSchema);
    tableInfo.setAggregateTableList(new ArrayList<TableSchema>());
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String schemaFilePath = carbonTablePath.getSchemaFilePath();
    String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
    tableInfo.setMetaDataFilepath(schemaMetadataPath);
    CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    org.apache.carbondata.format.TableInfo thriftTableInfo = schemaConverter.fromWrapperToExternalTableInfo(tableInfo, tableInfo.getDatabaseName(), tableInfo.getFactTable().getTableName());
    org.apache.carbondata.format.SchemaEvolutionEntry schemaEvolutionEntry = new org.apache.carbondata.format.SchemaEvolutionEntry(tableInfo.getLastUpdatedTime());
    thriftTableInfo.getFact_table().getSchema_evolution().getSchema_evolution_history().add(schemaEvolutionEntry);
    FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
    if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
        FileFactory.mkdirs(schemaMetadataPath, fileType);
    }
    ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
    thriftWriter.open();
    thriftWriter.write(thriftTableInfo);
    thriftWriter.close();
    return CarbonMetadata.getInstance().getCarbonTable(tableInfo.getTableUniqueName());
}