File tree Expand file tree Collapse file tree 1 file changed +0
-26
lines changed
Expand file tree Collapse file tree 1 file changed +0
-26
lines changed Original file line number Diff line number Diff line change @@ -681,32 +681,6 @@ def all_manifests(self, snapshots: Optional[list[Snapshot]] = None) -> "pa.Table
681681 )
682682 return pa .concat_tables (manifests_by_snapshots )
683683
684- def _all_known_files (self ) -> dict [str , set [str ]]:
685- """Get all the known files in the table.
686-
687- Returns:
688- dict of {file_type: set of file paths} for each file type.
689- """
690- snapshots = self .tbl .snapshots ()
691-
692- _all_known_files = {}
693- _all_known_files ["manifests" ] = set (self .all_manifests (snapshots )["path" ].to_pylist ())
694- _all_known_files ["manifest_lists" ] = {snapshot .manifest_list for snapshot in snapshots }
695- _all_known_files ["statistics" ] = {statistic .statistics_path for statistic in self .tbl .metadata .statistics }
696-
697- metadata_files = {entry .metadata_file for entry in self .tbl .metadata .metadata_log }
698- metadata_files .add (self .tbl .metadata_location ) # Include current metadata file
699- _all_known_files ["metadata" ] = metadata_files
700-
701- executor = ExecutorFactory .get_or_create ()
702- snapshot_ids = [snapshot .snapshot_id for snapshot in snapshots ]
703- files_by_snapshots : Iterator [Set [str ]] = executor .map (
704- lambda snapshot_id : set (self .files (snapshot_id )["file_path" ].to_pylist ()), snapshot_ids
705- )
706- _all_known_files ["datafiles" ] = reduce (set .union , files_by_snapshots , set ())
707-
708- return _all_known_files
709-
710684 def _all_files (self , data_file_filter : Optional [Set [DataFileContent ]] = None ) -> "pa.Table" :
711685 import pyarrow as pa
712686
You can’t perform that action at this time.
0 commit comments