Feature/saved search main (#103)

* Add saved search list and show commands (#95)

* Add saved search feature

* Added extraction commands

* Added tests

* Refactor: Format and privatize methods

* Added Changelog

* Added missing test for send-to

* Added usage note

* Refactor extraction commands at securitydata level

* Apply begin and end timestamp filter to saved search

* Apply begin and end timestamp filter to saved search

* Refactor/mutual exclusion saved search with advanced-query (#98)

* Fix: Mutual exclusion of saved search with advanced query

* Fix tests

* Refactor- Merge changes from master

* Privatize method and follow proper naming convention

* refactor- Use constants instead of strings

* Use public methods instead of private in tests

* Add constant

* Improved documentation

* Made saved-search mutually exclusive with other args alike advance-query

Author: kiran-chaudhary

CHANGELOG.md

@@ -40,6 +40,13 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta
 
 ### Added
 
+- Extraction subcommands of `code42 security-data`, `print/write-to/send-to` accepts argument `--saved-search` to
+   return saved search results.
+
+- `code42 security-data saved-search` commands:
+    - `list` prints out existing saved searches' id and name
+    - `show` takes a search id
+
 - `code42 high-risk-employee bulk` supports `add-risk-tags` and `remove-risk-tags`.
     - `code42 high-risk-employee bulk generate-template <cmd>` options `add-risk-tags` and `remove-risk-tags`.
         - `add-risk-tags` that takes a csv file with username and space separated risk tags.

README.md

@@ -61,6 +61,10 @@ To see all your profiles, do:
 code42 profile list
 ```
 
+A separate profile would be needed in order to keep the incremental checkpoints separate for different queries.
+i.e User needs to maintain separate profiles for file event queries and saved search queries as only one checkpoint
+is supported per profile.
+
 ## Security Data and Alerts
 
 Using the CLI, you can query for security events and alerts and send them to three possible destination types:

setup.py

@@ -24,8 +24,6 @@
         "c42eventextractor==0.3.2",
         "keyring==18.0.1",
         "keyrings.alt==3.2.0",
-        "py42>=1.2.0",
-        "pexpect>=4.8"
     ],
     license="MIT",
     include_package_data=True,
@@ -40,6 +38,7 @@
             "sphinx",
             "sphinx_rtd_theme",
             "tox==3.14.3",
+            "pexpect>=4.8",
         ]
     },
     classifiers=[

src/code42cli/cmds/alerts/extraction.py

@@ -16,7 +16,6 @@
 from code42cli.cmds.search_shared.extraction import (
     verify_begin_date_requirements,
     create_handlers,
-    exit_if_advanced_query_used_with_other_search_args,
     create_time_range_filter,
 )
 from code42cli.logger import get_main_cli_logger
@@ -40,7 +39,6 @@ def extract(sdk, profile, output_logger, args):
     handlers = create_handlers(sdk, AlertExtractor, output_logger, store)
     extractor = AlertExtractor(sdk, handlers)
     if args.advanced_query:
-        exit_if_advanced_query_used_with_other_search_args(args, enums.AlertFilterArguments())
         extractor.extract_advanced(args.advanced_query)
     else:
         verify_begin_date_requirements(args, store)

src/code42cli/cmds/alerts/main.py

@@ -1,5 +1,6 @@
 from code42cli.args import ArgConfig
 from code42cli.commands import Command, SubcommandLoader
+from code42cli.parser import exit_if_mutually_exclusive_args_used_together
 from code42cli.cmds.alerts.extraction import extract
 from code42cli.cmds.search_shared import args, logger_factory
 from code42cli.cmds.search_shared.enums import (
@@ -10,6 +11,9 @@
     RuleType,
 )
 from code42cli.cmds.search_shared.cursor_store import AlertCursorStore
+from code42cli.cmds.search_shared.args import (
+    create_incompatible_search_args, SEARCH_FOR_ALERTS
+)
 
 
 class MainAlertsSubcommandLoader(SubcommandLoader):
@@ -67,20 +71,31 @@ def clear_checkpoint(sdk, profile):
     AlertCursorStore(profile.name).replace_stored_cursor_timestamp(None)
 
 
+def _validate_args(args):
+    if args.advanced_query:
+        incompatible_search_args_dict = create_incompatible_search_args(SEARCH_FOR_ALERTS)
+        incompatible_search_args_list = list(incompatible_search_args_dict.keys())
+        invalid_args = incompatible_search_args_list + list(AlertFilterArguments())
+        exit_if_mutually_exclusive_args_used_together(args, invalid_args)
+
+
 def print_out(sdk, profile, args):
     """Activates 'print' command. It gets alerts and prints them to stdout."""
+    _validate_args(args)
     logger = logger_factory.get_logger_for_stdout(args.format)
     extract(sdk, profile, logger, args)
 
 
 def write_to(sdk, profile, args):
     """Activates 'write-to' command. It gets alerts and writes them to the given file."""
+    _validate_args(args)
     logger = logger_factory.get_logger_for_file(args.output_file, args.format)
     extract(sdk, profile, logger, args)
 
 
 def send_to(sdk, profile, args):
     """Activates 'send-to' command. It getsalerts and logs them to the given server."""
+    _validate_args(args)
     logger = logger_factory.get_logger_for_server(args.server, args.protocol, args.format)
     extract(sdk, profile, logger, args)
 
@@ -191,5 +206,5 @@ def _load_search_args(arg_collection):
             help=u"Filter alerts by description. Does fuzzy search by default.",
         ),
     }
-    search_args = args.create_search_args(search_for=u"alerts", filter_args=filter_args)
+    search_args = args.create_search_args(search_for=SEARCH_FOR_ALERTS, filter_args=filter_args)
     arg_collection.extend(search_args)

src/code42cli/cmds/search_shared/args.py

@@ -1,12 +1,16 @@
 from code42cli.cmds.search_shared.enums import SearchArguments, OutputFormat, AlertOutputFormat
 from code42cli.args import ArgConfig
 
+SEARCH_FOR_ALERTS = u"alerts"
+SEARCH_FOR_FILE_EVENTS = u"file events"
 
-def create_search_args(search_for, filter_args):
-    advanced_query_incompatible_args = create_advanced_query_incompatible_search_args(search_for)
-    filter_args.update(advanced_query_incompatible_args)
 
-    format_enum = AlertOutputFormat() if search_for == "alerts" else OutputFormat()
+def create_search_args(search_for, filter_args):
+    incompatible_args = create_incompatible_search_args(search_for)
+    filter_args.update(incompatible_args)
+    if search_for == SEARCH_FOR_FILE_EVENTS:
+        filter_args.update(_saved_search_args())
+    format_enum = AlertOutputFormat() if search_for == SEARCH_FOR_ALERTS else OutputFormat()
 
     advanced_query_compatible_args = {
         SearchArguments.ADVANCED_QUERY: ArgConfig(
@@ -31,7 +35,16 @@ def create_search_args(search_for, filter_args):
     return filter_args
 
 
-def create_advanced_query_incompatible_search_args(search_for=None):
+def _saved_search_args():
+    saved_search = ArgConfig(
+        u"--saved-search",
+        help=u"Limits events to those discoverable with the saved search "
+             u"filters for the saved search with the given ID.\n"
+             u"WARNING: Using saved search is incompatible with other query-building args.")
+    return {u"saved_search": saved_search}
+
+
+def create_incompatible_search_args(search_for=None):
     """Returns a dict of args that are incompatible with the --advanced-query flag. Any new 
     incompatible args should go here as this is function is also used for arg validation."""
     args = {
@@ -60,3 +73,13 @@ def create_advanced_query_incompatible_search_args(search_for=None):
         ),
     }
     return args
+
+
+def get_advanced_query_incompatible_search_args(search_for):
+    incompatible_args = create_incompatible_search_args(search_for)
+    incompatible_args.update(_saved_search_args())
+    return incompatible_args
+
+
+def get_saved_search_incompatible_search_args(search_for):
+    return create_incompatible_search_args(search_for)

src/code42cli/cmds/search_shared/extraction.py

@@ -8,7 +8,6 @@
 from code42cli.logger import get_main_cli_logger
 from code42cli.cmds.alerts.util import get_alert_details
 from code42cli.util import warn_interrupt
-from code42cli.cmds.search_shared.args import create_advanced_query_incompatible_search_args
 
 logger = get_main_cli_logger()
 
@@ -74,18 +73,6 @@ def handle_response(response):
     return handlers
 
 
-def exit_if_advanced_query_used_with_other_search_args(args, search_arg_enum):
-    incompatible_search_args_dict = create_advanced_query_incompatible_search_args()
-    incompatible_search_args_list = list(incompatible_search_args_dict.keys())
-    invalid_args = incompatible_search_args_list + list(search_arg_enum)
-    for arg in invalid_args:
-        if args.__dict__[arg]:
-            logger.print_and_log_error(
-                u"You cannot use --advanced-query with additional search args."
-            )
-            exit(1)
-
-
 def create_time_range_filter(filter_cls, begin_date=None, end_date=None):
     """Creates a filter using the given filter class (must be a subclass of 
         :class:`py42.sdk.queries.query_filter.QueryFilterTimestampField`) and date args. Returns 

src/code42cli/cmds/securitydata/extraction.py

@@ -3,13 +3,11 @@
 
 from code42cli.cmds.search_shared.enums import (
     ExposureType as ExposureTypeOptions,
-    FileEventFilterArguments,
 )
 from code42cli.cmds.search_shared.cursor_store import FileEventCursorStore
 from code42cli.cmds.search_shared.extraction import (
     verify_begin_date_requirements,
     create_handlers,
-    exit_if_advanced_query_used_with_other_search_args,
     create_time_range_filter,
 )
 import code42cli.errors as errors
@@ -18,7 +16,7 @@
 logger = get_main_cli_logger()
 
 
-def extract(sdk, profile, output_logger, args):
+def extract(sdk, profile, output_logger, args, query=None):
     """Extracts file events using the given command-line arguments.
 
         Args:
@@ -29,18 +27,21 @@ def extract(sdk, profile, output_logger, args):
                 write-to: uses a logger that logs to a file.
                 send-to: uses a logger that sends logs to a server.
             args: Command line args used to build up file event query filters.
+            query: FileEventQuery instance created from search-id of saved search.
     """
     store = FileEventCursorStore(profile.name) if args.incremental else None
     handlers = create_handlers(sdk, FileEventExtractor, output_logger, store)
     extractor = FileEventExtractor(sdk, handlers)
-    if args.advanced_query:
-        exit_if_advanced_query_used_with_other_search_args(args, FileEventFilterArguments())
-        extractor.extract_advanced(args.advanced_query)
-    else:
+    if not args.advanced_query and not args.saved_search:
         verify_begin_date_requirements(args, store)
         if args.type:
             _verify_exposure_types(args.type)
+    if args.advanced_query:
+        extractor.extract_advanced(args.advanced_query)
+    else:
         filters = _create_file_event_filters(args)
+        if args.saved_search:
+            filters.extend(query._filter_group_list)
         extractor.extract(*filters)
     if handlers.TOTAL_EVENTS == 0 and not errors.ERRORED:
         logger.print_info(u"No results found.")

src/code42cli/cmds/securitydata/main.py

@@ -1,4 +1,5 @@
 from code42cli.args import ArgConfig
+from code42cli.parser import exit_if_mutually_exclusive_args_used_together
 from code42cli.cmds.search_shared import logger_factory, args
 from code42cli.cmds.search_shared.enums import (
     FileEventFilterArguments,
@@ -8,13 +9,20 @@
 from code42cli.cmds.securitydata.extraction import extract
 from code42cli.cmds.search_shared.cursor_store import FileEventCursorStore
 from code42cli.commands import Command, SubcommandLoader
+from code42cli.cmds.securitydata.savedsearch.commands import SavedSearchSubCommandLoader
+from code42cli.cmds.search_shared.args import (
+    SEARCH_FOR_FILE_EVENTS,
+    get_advanced_query_incompatible_search_args,
+    get_saved_search_incompatible_search_args,
+)
 
 
 class SecurityDataSubcommandLoader(SubcommandLoader):
     PRINT = u"print"
     WRITE_TO = u"write-to"
     SEND_TO = u"send-to"
     CLEAR_CHECKPOINT = u"clear-checkpoint"
+    SAVED_SEARCH = u"saved-search"
 
     def load_commands(self):
         """Sets up the `security-data` subcommand with all of its subcommands."""
@@ -54,7 +62,14 @@ def load_commands(self):
             handler=clear_checkpoint,
         )
 
-        return [print_func, write, send, clear]
+        saved_search = Command(
+            self.SAVED_SEARCH,
+            u"Manage saved searches.",
+            subcommand_loader=SavedSearchSubCommandLoader(self.SAVED_SEARCH)
+
+        )
+
+        return [print_func, write, send, clear, saved_search]
 
 
 def clear_checkpoint(sdk, profile):
@@ -65,22 +80,48 @@ def clear_checkpoint(sdk, profile):
     FileEventCursorStore(profile.name).replace_stored_cursor_timestamp(None)
 
 
+def _get_incompatible_search_args(incompatible_search_args_dict):
+    incompatible_search_args_list = list(incompatible_search_args_dict.keys())
+    return incompatible_search_args_list + list(FileEventFilterArguments())
+
+
+def _validate_args(args):
+
+    if args.advanced_query:
+        incompatible_search_args_dict = get_advanced_query_incompatible_search_args(SEARCH_FOR_FILE_EVENTS)
+        incompatible_search_args = _get_incompatible_search_args(incompatible_search_args_dict)
+        exit_if_mutually_exclusive_args_used_together(args, incompatible_search_args)
+    if args.saved_search:
+        incompatible_search_args_dict = get_saved_search_incompatible_search_args(SEARCH_FOR_FILE_EVENTS)
+        incompatible_search_args = _get_incompatible_search_args(incompatible_search_args_dict)
+        exit_if_mutually_exclusive_args_used_together(args, incompatible_search_args, u"--saved-search")
+
+
+def _extract(sdk, profile, logger, args):
+    query = sdk.securitydata.savedsearches.get_query(args.saved_search) \
+        if args.saved_search else None
+    extract(sdk, profile, logger, args, query)
+
+
 def print_out(sdk, profile, args):
     """Activates 'print' command. It gets security events and prints them to stdout."""
+    _validate_args(args)
     logger = logger_factory.get_logger_for_stdout(args.format)
-    extract(sdk, profile, logger, args)
+    _extract(sdk, profile, logger, args)
 
 
 def write_to(sdk, profile, args):
     """Activates 'write-to' command. It gets security events and writes them to the given file."""
+    _validate_args(args)
     logger = logger_factory.get_logger_for_file(args.output_file, args.format)
-    extract(sdk, profile, logger, args)
+    _extract(sdk, profile, logger, args)
 
 
 def send_to(sdk, profile, args):
     """Activates 'send-to' command. It gets security events and logs them to the given server."""
+    _validate_args(args)
     logger = logger_factory.get_logger_for_server(args.server, args.protocol, args.format)
-    extract(sdk, profile, logger, args)
+    _extract(sdk, profile, logger, args)
 
 
 def _load_write_to_args(arg_collection):
@@ -166,5 +207,5 @@ def _load_search_args(arg_collection):
             help=u"Get all events including non-exposure events.",
         ),
     }
-    search_args = args.create_search_args(search_for=u"file events", filter_args=filter_args)
+    search_args = args.create_search_args(search_for=SEARCH_FOR_FILE_EVENTS, filter_args=filter_args)
     arg_collection.extend(search_args)