2020"""
2121
2222
23+ import glob
2324import yaml
2425import os
2526import shutil
@@ -153,6 +154,81 @@ def copy_markdown(self):
153154 f"_build/html/docfx_yaml" ,
154155 )
155156
157+ def validate_section (self , toc ):
158+ # Make sure each rst file is listed in the toc.
159+ items_in_toc = [
160+ d ["items" ] for d in toc [0 ]["items" ] if d ["name" ] == self .title and ".rst"
161+ ][0 ]
162+ items_in_dir = [f for f in os .listdir (self .dir_name ) if f .endswith (".rst" )]
163+ # subtract 1 for index
164+ assert len (items_in_toc ) == len (items_in_dir ) - 1
165+ for file in items_in_dir :
166+ if file != self .index_file_name :
167+ base_name , _ = os .path .splitext (file )
168+ assert any (d ["href" ] == f"{ base_name } for d in items_in_toc )
169+ # make sure the markdown files are present in the docfx_yaml directory
170+ md_files = [d ["href" ] for d in items_in_toc ]
171+ for file in md_files :
172+ assert os .path .exists (f"_build/html/docfx_yaml/{ file } )
173+
174+
175+ class UIDFilteredTocSection (TocSection ):
176+ def __init__ (self , toc_file_path , section_name , title , uid_prefix ):
177+ """Creates a filtered section denoted by section_name in the toc_file_path to items with the given UID prefix.
178+
179+ The section is then renamed to the title.
180+ """
181+ current_toc = yaml .safe_load (open (toc_file_path , "r" ))
182+ self .uid_prefix = uid_prefix
183+
184+ # Since we are looking for a specific section_name there should only
185+ # be one match.
186+ section_items = [
187+ d for d in current_toc [0 ]["items" ] if d ["name" ] == section_name
188+ ][0 ]["items" ]
189+ filtered_items = [d for d in section_items if d ["uid" ].startswith (uid_prefix )]
190+ self .items = filtered_items
191+ self .title = title
192+
193+ def copy_markdown (self ):
194+ """
195+ No-op because we are filtering on UIDs, not markdown files.
196+ """
197+ pass
198+
199+ def validate_section (self , toc ):
200+ uids_in_toc = set ()
201+
202+ # A UID-filtered TOC tree looks like the following:
203+ # - items:
204+ # <optional> items: <more stuff>
205+ # name: <name>
206+ # uid: <fully qualified path to a class>
207+ #
208+ # Walk through the TOC tree to find all UIDs recursively.
209+ def find_uids_in_items (items ):
210+ uids_in_toc .add (items ["uid" ])
211+ for subitem in items .get ("items" , []):
212+ find_uids_in_items (subitem )
213+
214+ items_in_toc = [d ["items" ] for d in toc [0 ]["items" ] if d ["name" ] == self .title ][
215+ 0
216+ ]
217+ for item in items_in_toc :
218+ find_uids_in_items (item )
219+
220+ # Now that we have all the UIDs, first match all of them
221+ # with corresponding .yml files.
222+ for uid in uids_in_toc :
223+ assert os .path .exists (f"_build/html/docfx_yaml/{ uid } )
224+
225+ # Also validate that every uid yml file that starts with the uid_prefix
226+ # exists in the section.
227+ for filename in glob .glob (
228+ f"{ self .uid_prefix } , root_dir = "_build/html/docfx_yaml"
229+ ):
230+ assert filename [:- 4 ] in uids_in_toc
231+
156232
157233def validate_toc (toc_file_path , expected_section_list , added_sections ):
158234 current_toc = yaml .safe_load (open (toc_file_path , "r" ))
@@ -164,43 +240,27 @@ def validate_toc(toc_file_path, expected_section_list, added_sections):
164240 # make sure each customs ection is in the toc
165241 for section in added_sections :
166242 assert section .title in found_sections
167- # make sure each rst file in each custom section dir is listed in the toc
168- for section in added_sections :
169- items_in_toc = [
170- d ["items" ]
171- for d in current_toc [0 ]["items" ]
172- if d ["name" ] == section .title and ".rst"
173- ][0 ]
174- items_in_dir = [f for f in os .listdir (section .dir_name ) if f .endswith (".rst" )]
175- # subtract 1 for index
176- assert len (items_in_toc ) == len (items_in_dir ) - 1
177- for file in items_in_dir :
178- if file != section .index_file_name :
179- base_name , _ = os .path .splitext (file )
180- assert any (d ["href" ] == f"{ base_name } for d in items_in_toc )
181- # make sure the markdown files are present in the docfx_yaml directory
182- for section in added_sections :
183- items_in_toc = [
184- d ["items" ]
185- for d in current_toc [0 ]["items" ]
186- if d ["name" ] == section .title and ".rst"
187- ][0 ]
188- md_files = [d ["href" ] for d in items_in_toc ]
189- for file in md_files :
190- assert os .path .exists (f"_build/html/docfx_yaml/{ file } )
243+ section .validate_section (current_toc )
191244 print ("Toc validation passed" )
192245
193246
194247if __name__ == "__main__" :
195248 # Add secrtions for the async_data_client and classic_client directories
196249 toc_path = "_build/html/docfx_yaml/toc.yml"
250+
197251 custom_sections = [
198252 TocSection (dir_name = "data_client" , index_file_name = "data_client_usage.rst" ),
253+ UIDFilteredTocSection (
254+ toc_file_path = toc_path ,
255+ section_name = "Bigtable Admin V2" ,
256+ title = "Admin Client" ,
257+ uid_prefix = "google.cloud.bigtable_admin_v2" ,
258+ ),
199259 TocSection (dir_name = "classic_client" , index_file_name = "usage.rst" ),
200260 ]
201261 add_sections (toc_path , custom_sections )
202262 # Remove the Bigtable section, since it has duplicated data
203- remove_sections (toc_path , ["Bigtable" ])
263+ remove_sections (toc_path , ["Bigtable" , "Bigtable Admin V2" ])
204264 # run validation to make sure yaml is structured as we expect
205265 validate_toc (
206266 toc_file_path = toc_path ,
@@ -210,6 +270,7 @@ def validate_toc(toc_file_path, expected_section_list, added_sections):
210270 "Changelog" ,
211271 "Multiprocessing" ,
212272 "Data Client" ,
273+ "Admin Client" ,
213274 "Classic Client" ,
214275 ],
215276 added_sections = custom_sections ,
0 commit comments