Examples with MetadataVersion org.apache.drill.exec.store.parquet.metadata.MetadataVersion used on opensource projects

Example 1 with MetadataVersion

use of org.apache.drill.exec.store.parquet.metadata.MetadataVersion in project drill by apache.

the class TestParquetMetadataCache method testFutureUnsupportedMetadataVersion.

@Test
@Category(UnlikelyTest.class)
public void testFutureUnsupportedMetadataVersion() throws Exception {
    final String unsupportedMetadataVersion = "unsupported_metadata_version";
    try {
        test("use dfs.tmp");
        test("create table `%s` as select * from cp.`tpch/nation.parquet`", unsupportedMetadataVersion);
        MetadataVersion lastVersion = MetadataVersion.Constants.SUPPORTED_VERSIONS.last();
        // Get the future version, which is absent in MetadataVersions.SUPPORTED_VERSIONS set
        String futureVersion = new MetadataVersion(lastVersion.getMajor() + 1, 0).toString();
        File metaDataFile = dirTestWatcher.copyResourceToTestTmp(Paths.get("parquet", "unsupported_metadata", "unsupported_metadata_version.requires_replace.txt"), Paths.get(unsupportedMetadataVersion, Metadata.OLD_METADATA_FILENAME));
        dirTestWatcher.replaceMetaDataContents(metaDataFile, dirTestWatcher.getDfsTestTmpDir(), futureVersion);
        String query = String.format("select * from %s", unsupportedMetadataVersion);
        int expectedRowCount = 25;
        int expectedNumFiles = 1;
        int actualRowCount = testSql(query);
        assertEquals("An incorrect result was obtained while querying a table with metadata cache files", expectedRowCount, actualRowCount);
        String numFilesPattern = "numFiles=" + expectedNumFiles;
        // ignoring metadata cache file
        String usedMetaPattern = "usedMetadataFile=false";
        PlanTestBase.testPlanMatchingPatterns(query, new String[] { numFilesPattern, usedMetaPattern }, new String[] { "Filter" });
    } finally {
        test("drop table if exists %s", unsupportedMetadataVersion);
    }
}

Example 2 with MetadataVersion

use of org.apache.drill.exec.store.parquet.metadata.MetadataVersion in project drill by apache.

the class ParquetTableMetadataUtils method getIntermediateFields.

/**
 * Returns map of column names with their Drill types for every {@code NameSegment} in {@code SchemaPath}
 * in specified {@code rowGroup}. The type for a {@code SchemaPath} can be {@code null} in case when
 * it is not possible to determine its type. Actually, as of now this hierarchy is of interest solely
 * because there is a need to account for {@link org.apache.drill.common.types.TypeProtos.MinorType#DICT}
 * to make sure filters used on {@code DICT}'s values (get by key) are not pruned out before actual filtering
 * happens.
 *
 * @param parquetTableMetadata the source of column types
 * @param rowGroup row group whose columns should be discovered
 * @return map of column names with their drill types
 */
public static Map<SchemaPath, TypeProtos.MajorType> getIntermediateFields(MetadataBase.ParquetTableMetadataBase parquetTableMetadata, MetadataBase.RowGroupMetadata rowGroup) {
    Map<SchemaPath, TypeProtos.MajorType> columns = new LinkedHashMap<>();
    MetadataVersion metadataVersion = new MetadataVersion(parquetTableMetadata.getMetadataVersion());
    boolean hasParentTypes = metadataVersion.isAtLeast(4, 1);
    if (!hasParentTypes) {
        return Collections.emptyMap();
    }
    for (MetadataBase.ColumnMetadata column : rowGroup.getColumns()) {
        Metadata_V4.ColumnTypeMetadata_v4 columnTypeMetadata = ((Metadata_V4.ParquetTableMetadata_v4) parquetTableMetadata).getColumnTypeInfo(column.getName());
        List<OriginalType> parentTypes = columnTypeMetadata.parentTypes;
        List<TypeProtos.MajorType> drillTypes = ParquetReaderUtility.getComplexTypes(parentTypes);
        for (int i = 0; i < drillTypes.size(); i++) {
            SchemaPath columnPath = SchemaPath.getCompoundPath(i + 1, column.getName());
            TypeProtos.MajorType drillType = drillTypes.get(i);
            putType(columns, columnPath, drillType);
        }
    }
    return columns;
}

Example 3 with MetadataVersion

use of org.apache.drill.exec.store.parquet.metadata.MetadataVersion in project drill by apache.

the class TestParquetMetadataVersion method testWithoutMinorVersion.

@Test
public void testWithoutMinorVersion() throws Exception {
    MetadataVersion withoutMinorVersion = new MetadataVersion("v3");
    MetadataVersion expectedVersionWithoutMinorVersion = new MetadataVersion(3, 0);
    assertEquals("Parquet metadata version is parsed incorrectly", expectedVersionWithoutMinorVersion, withoutMinorVersion);
}

Example 4 with MetadataVersion

use of org.apache.drill.exec.store.parquet.metadata.MetadataVersion in project drill by apache.

the class TestParquetMetadataVersion method testIsEqual.

@Test
public void testIsEqual() {
    MetadataVersion version = new MetadataVersion(3, 2);
    assertTrue(version.isEqualTo(3, 2));
    assertFalse(version.isEqualTo(4, 2));
    assertFalse(version.isEqualTo(2, 3));
    assertFalse(version.isEqualTo(1, 0));
    assertFalse(version.isEqualTo(3, 1));
    assertFalse(version.isEqualTo(1, 2));
}

Example 5 with MetadataVersion

use of org.apache.drill.exec.store.parquet.metadata.MetadataVersion in project drill by apache.

the class TestParquetMetadataVersion method testFirstLetter.

@Test
public void testFirstLetter() throws Exception {
    MetadataVersion versionWithFirstLetter = new MetadataVersion("v4");
    MetadataVersion expectedVersion = new MetadataVersion(4, 0);
    assertEquals("Parquet metadata version is parsed incorrectly", expectedVersion, versionWithFirstLetter);
    MetadataVersion versionWithoutFirstLetter = new MetadataVersion("4");
    assertEquals("Parquet metadata version is parsed incorrectly", expectedVersion, versionWithoutFirstLetter);
}