Feat: support recursive indexing for index function

Author: ligangty

charon/cmd/cmd_index.py

@@ -42,6 +42,13 @@
     """,
     required=True
 )
+@option(
+    "--recursive",
+    "-r",
+    help="If do indexing recursively under $path",
+    is_flag=True,
+    default=False
+)
 @option(
     "--config",
     "-c",
@@ -69,6 +76,7 @@
 def index(
     path: str,
     target: str,
+    recursive: bool = False,
     config: str = None,
     debug: bool = False,
     quiet: bool = False,
@@ -120,7 +128,15 @@ def index(
             if not aws_bucket:
                 logger.error("No bucket specified for target %s!", target)
             else:
-                re_index(b, path, package_type, aws_profile, dryrun)
+                args = {
+                    "target": b,
+                    "path": path,
+                    "package_type": package_type,
+                    "aws_profile": aws_profile,
+                    "recursive": recursive,
+                    "dry_run": dryrun
+                }
+                re_index(**args)  # type: ignore
 
     except Exception:
         print(traceback.format_exc())

charon/pkgs/indexing.py

@@ -266,7 +266,7 @@ def re_index(
     path: str,
     package_type: str,
     aws_profile: str = None,
-    # cf_enable: bool = False,
+    recursive: bool = False,
     dry_run: bool = False
 ):
     """Refresh the index.html for the specified folder in the bucket.
@@ -306,17 +306,31 @@ def re_index(
         logger.debug("The re-indexed page content: %s", index_content)
         if not dry_run:
             index_path = os.path.join(path, "index.html")
+            logger.info("Start re-indexing %s in bucket %s", index_path, bucket_name)
             if path == "/":
                 index_path = "index.html"
             s3_client.simple_delete_file(index_path, (bucket_name, real_prefix))
             s3_client.simple_upload_file(
                 index_path, index_content, (bucket_name, real_prefix),
                 "text/html", digest_content(index_content)
             )
-            # We will not invalidate index.html per cost consideration
-            # if cf_enable:
-            #     cf_client = CFClient(aws_profile=aws_profile)
-            #     invalidate_cf_paths(cf_client, bucket, [index_path])
+            logger.info("%s re-indexing finished", index_path)
+        if recursive:
+            for c in contents:
+                if c.endswith("/"):
+                    sub_path = c.removeprefix(real_prefix).strip()
+                    if sub_path.startswith("/"):
+                        sub_path = sub_path.removeprefix("/")
+                    logger.debug("subpath: %s", sub_path)
+                    args = {
+                        "target": target,
+                        "path": sub_path,
+                        "package_type": package_type,
+                        "aws_profile": aws_profile,
+                        "recursive": recursive,
+                        "dry_run": dry_run
+                    }
+                    re_index(**args)  # type: ignore
     else:
         logger.warning(
             "The path %s does not contain any contents in bucket %s. "