Apache Parquet 格式简介

 1/// Description for ColumnIndex.
 2/// Each <array-field>[i] refers to the page at OffsetIndex.page_locations[i]
 3#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
 4pub struct ColumnIndex {
 5  /// A list of Boolean values to determine the validity of the corresponding
 6  /// min and max values. If true, a page contains only null values, and writers
 7  /// have to set the corresponding entries in min_values and max_values to
 8  /// byte[0], so that all lists have the same length. If false, the
 9  /// corresponding entries in min_values and max_values must be valid.
10  pub null_pages: Vec<bool>,
11  /// Two lists containing lower and upper bounds for the values of each page.
12  /// These may be the actual minimum and maximum values found on a page, but
13  /// can also be (more compact) values that do not exist on a page. For
14  /// example, instead of storing ""Blart Versenwald III", a writer may set
15  /// min_values[i]="B", max_values[i]="C". Such more compact values must still
16  /// be valid values within the column's logical type. Readers must make sure
17  /// that list entries are populated before using them by inspecting null_pages.
18  pub min_values: Vec<Vec<u8>>,
19  pub max_values: Vec<Vec<u8>>,
20  /// Stores whether both min_values and max_values are orderd and if so, in
21  /// which direction. This allows readers to perform binary searches in both
22  /// lists. Readers cannot assume that max_values[i] <= min_values[i+1], even
23  /// if the lists are ordered.
24  pub boundary_order: BoundaryOrder,
25  /// A list containing the number of null values for each page *
26  pub null_counts: Option<Vec<i64>>,
27}
28
29
30#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
31pub struct OffsetIndex {
32  /// PageLocations, ordered by increasing PageLocation.offset. It is required
33  /// that page_locations[i].first_row_index < page_locations[i+1].first_row_index.
34  pub page_locations: Vec<PageLocation>,
35}