Chore/allow optional cols for bulk cmds (#352)

* switching flat file bulk commands to use csv_args

* test for flat file compatability

* update file_reader

* strip commented header line from files

* update user guide

* remove flat file methods

* typo

Author: tora-kozic

docs/guides.md

@@ -11,3 +11,4 @@
 * [Configure Trusted Activities](userguides/trustedactivities.md)
 * [Configure Alert Rules](userguides/alertrules.md)
 * [Add and Manage Cases](userguides/cases.md)
+* [Use Bulk Commands](userguides/bulkcommands.md)

docs/userguides/bulkcommands.md

@@ -0,0 +1,23 @@
+# Using Bulk Commands
+
+Bulk functionality is available for many Code42 CLI methods, more details on which commands have bulk capabilities can be found in the [Commands Documentation](../commands.md).
+
+All bulk methods take a CSV file as input.
+
+The `generate-template` command can be used to create a CSV file with the necessary headers for a particular command.
+
+For instance, the following command will create a file named `devices_bulk_deactivate.csv` with a single column header row of `guid`.
+```bash
+code42 devices bulk generate-template deactivate
+```
+
+The CSV file can contain more columns than are necessary for the command, however then the header row is **required**.
+
+If the CSV file contains the *exact* number of columns that are necessary for the command then the header row is **optional**, but columns are expected to be in the same order as the template.
+
+To run a bulk method, simply pass the CSV file path to the desired command.  For example, you would use to following command to deactivate multiple devices within your organization at once:
+
+
+```bash
+code42 devices bulk deactivate devices_bulk_deactivate.csv
+```

src/code42cli/bulk.py

@@ -17,14 +17,9 @@ def __iter__(self):
         return iter([self.ADD, self.REMOVE])
 
 
-def write_template_file(path, columns=None, flat_item=None):
+def write_template_file(path, columns):
     with open(path, "w", encoding="utf8") as new_file:
-        if columns:
-            new_file.write(",".join(columns))
-        else:
-            new_file.write(
-                f"# This template takes a single {flat_item or 'item'} to be processed on each row."
-            )
+        new_file.write(",".join(columns))
 
 
 def generate_template_cmd_factory(group_name, commands_dict, help_message=None):
@@ -58,10 +53,7 @@ def generate_template(cmd, path):
         if not path:
             filename = f"{group_name}_bulk_{cmd.replace('-', '_')}.csv"
             path = os.path.join(os.getcwd(), filename)
-        if isinstance(columns, str):
-            write_template_file(path, columns=None, flat_item=columns)
-        else:
-            write_template_file(path, columns=columns)
+        write_template_file(path, columns)
 
     return generate_template
 
@@ -148,10 +140,7 @@ def run(self):
         return self._stats._results
 
     def _process_row(self, row):
-        if isinstance(row, dict):
-            self._process_csv_row(row)
-        elif row:
-            self._process_flat_file_row(row.strip())
+        self._process_csv_row(row)
 
     def _process_csv_row(self, row):
         # Removes problems from including extra columns. Error messages from out of order args
@@ -163,12 +152,6 @@ def _process_csv_row(self, row):
             lambda *args, **kwargs: self._handle_row(*args, **kwargs), **row_values
         )
 
-    def _process_flat_file_row(self, row):
-        if row:
-            self.__worker.do_async(
-                lambda *args, **kwargs: self._handle_row(*args, **kwargs), row
-            )
-
     def _handle_row(self, *args, **kwargs):
         return self._row_handler(*args, **kwargs)
 

src/code42cli/cmds/departing_employee.py

@@ -15,7 +15,6 @@
 from code42cli.cmds.shared import get_user_id
 from code42cli.errors import Code42CLIError
 from code42cli.file_readers import read_csv_arg
-from code42cli.file_readers import read_flat_file_arg
 from code42cli.options import format_option
 from code42cli.options import sdk_options
 
@@ -86,16 +85,21 @@ def bulk(state):
 
 DEPARTING_EMPLOYEE_CSV_HEADERS = ["username", "cloud_alias", "departure_date", "notes"]
 
+REMOVE_EMPLOYEE_HEADERS = ["username"]
+
 departing_employee_generate_template = generate_template_cmd_factory(
     group_name="departing_employee",
-    commands_dict={"add": DEPARTING_EMPLOYEE_CSV_HEADERS, "remove": "username"},
+    commands_dict={
+        "add": DEPARTING_EMPLOYEE_CSV_HEADERS,
+        "remove": REMOVE_EMPLOYEE_HEADERS,
+    },
 )
 bulk.add_command(departing_employee_generate_template)
 
 
 @bulk.command(
     name="add",
-    help="Bulk add users to the Departing Employees detection list using a CSV file with "
+    help="Bulk add users to the departing employees detection list using a CSV file with "
     f"format: {','.join(DEPARTING_EMPLOYEE_CSV_HEADERS)}.",
 )
 @read_csv_arg(headers=DEPARTING_EMPLOYEE_CSV_HEADERS)
@@ -125,20 +129,19 @@ def handle_row(username, cloud_alias, departure_date, notes):
 
 @bulk.command(
     name="remove",
-    help="Bulk remove users from the Departing Employees detection list using a line-separated "
-    "file of usernames.",
+    help=f"Bulk remove users from the departing employees detection list using a CSV file with format {','.join(REMOVE_EMPLOYEE_HEADERS)}.",
 )
-@read_flat_file_arg
+@read_csv_arg(headers=REMOVE_EMPLOYEE_HEADERS)
 @sdk_options()
-def bulk_remove(state, file_rows):
+def bulk_remove(state, csv_rows):
     sdk = state.sdk
 
     def handle_row(username):
         _remove_departing_employee(sdk, username)
 
     run_bulk_process(
         handle_row,
-        file_rows,
+        csv_rows,
         progress_label="Removing users from the Departing Employees detection list:",
     )
 

src/code42cli/cmds/devices.py

@@ -566,7 +566,6 @@ def bulk(state):
 _bulk_device_activation_headers = ["guid"]
 _bulk_device_rename_headers = ["guid", "name"]
 
-
 devices_generate_template = generate_template_cmd_factory(
     group_name="devices",
     commands_dict={

src/code42cli/cmds/high_risk_employee.py

@@ -18,7 +18,6 @@
 from code42cli.cmds.detectionlists.options import username_arg
 from code42cli.cmds.shared import get_user_id
 from code42cli.file_readers import read_csv_arg
-from code42cli.file_readers import read_flat_file_arg
 from code42cli.options import format_option
 from code42cli.options import sdk_options
 
@@ -110,12 +109,13 @@ def bulk(state):
 
 HIGH_RISK_EMPLOYEE_CSV_HEADERS = ["username", "cloud_alias", "risk_tag", "notes"]
 RISK_TAG_CSV_HEADERS = ["username", "tag"]
+REMOVE_EMPLOYEE_HEADERS = ["username"]
 
 high_risk_employee_generate_template = generate_template_cmd_factory(
     group_name="high_risk_employee",
     commands_dict={
         "add": HIGH_RISK_EMPLOYEE_CSV_HEADERS,
-        "remove": "username",
+        "remove": REMOVE_EMPLOYEE_HEADERS,
         "add-risk-tags": RISK_TAG_CSV_HEADERS,
         "remove-risk-tags": RISK_TAG_CSV_HEADERS,
     },
@@ -145,20 +145,19 @@ def handle_row(username, cloud_alias, risk_tag, notes):
 
 @bulk.command(
     name="remove",
-    help="Bulk remove users from the high risk employees detection list using a line-separated file "
-    "of usernames.",
+    help=f"Bulk remove users from the high risk employees detection list using a CSV file with format {','.join(REMOVE_EMPLOYEE_HEADERS)}.",
 )
-@read_flat_file_arg
+@read_csv_arg(headers=REMOVE_EMPLOYEE_HEADERS)
 @sdk_options()
-def bulk_remove(state, file_rows):
+def bulk_remove(state, csv_rows):
     sdk = state.sdk
 
     def handle_row(username):
         _remove_high_risk_employee(sdk, username)
 
     run_bulk_process(
         handle_row,
-        file_rows,
+        csv_rows,
         progress_label="Removing users from high risk employee detection list:",
     )
 

src/code42cli/file_readers.py

@@ -28,6 +28,12 @@ def read_csv(file, headers):
     else error is raised.
     """
     lines = file.readlines()
+
+    # check if header is commented for flat-file backwards compatability
+    if lines[0].startswith("#"):
+        # strip comment line
+        lines.pop(0)
+
     first_line = lines[0].strip().split(",")
 
     # handle when first row has all of our expected headers
@@ -52,21 +58,3 @@ def read_csv(file, headers):
     else:
         missing = [field for field in headers if field not in first_line]
         raise Code42CLIError(f"Missing required columns in csv: {missing}")
-
-
-def read_flat_file(file):
-    """Helper to read rows of a flat file, automatically removing header comment row if
-    it exists, and strips whitespace from each row automatically."""
-    first_row = next(file)
-    if first_row.startswith("#"):
-        return [row.strip() for row in file]
-    else:
-        return [first_row.strip(), *[row.strip() for row in file]]
-
-
-read_flat_file_arg = click.argument(
-    "file_rows",
-    type=AutoDecodedFile("r"),
-    metavar="FILE",
-    callback=lambda ctx, param, arg: read_flat_file(arg),
-)

tests/cmds/test_departing_employee.py

@@ -277,13 +277,76 @@ def test_remove_bulk_users_uses_expected_arguments(runner, mocker, cli_state_wit
     bulk_processor = mocker.patch("code42cli.cmds.departing_employee.run_bulk_process")
     with runner.isolated_filesystem():
         with open("test_remove.csv", "w") as csv:
-            csv.writelines(["# username\n", "test_user1\n", "test_user2\n"])
+            csv.writelines(["username\n", "test_user1\n", "test_user2\n"])
+        runner.invoke(
+            cli,
+            ["departing-employee", "bulk", "remove", "test_remove.csv"],
+            obj=cli_state_with_user,
+        )
+    assert bulk_processor.call_args[0][1] == [
+        {"username": "test_user1"},
+        {"username": "test_user2"},
+    ]
+
+
+def test_remove_bulk_users_uses_expected_arguments_when_no_header(
+    runner, mocker, cli_state_with_user
+):
+    bulk_processor = mocker.patch("code42cli.cmds.departing_employee.run_bulk_process")
+    with runner.isolated_filesystem():
+        with open("test_remove.csv", "w") as csv:
+            csv.writelines(["test_user1\n", "test_user2\n"])
+        runner.invoke(
+            cli,
+            ["departing-employee", "bulk", "remove", "test_remove.csv"],
+            obj=cli_state_with_user,
+        )
+    assert bulk_processor.call_args[0][1] == [
+        {"username": "test_user1"},
+        {"username": "test_user2"},
+    ]
+
+
+def test_remove_bulk_users_uses_expected_arguments_when_extra_columns(
+    runner, mocker, cli_state_with_user
+):
+    bulk_processor = mocker.patch("code42cli.cmds.departing_employee.run_bulk_process")
+    with runner.isolated_filesystem():
+        with open("test_remove.csv", "w") as csv:
+            csv.writelines(
+                [
+                    "username,test_column\n",
+                    "test_user1,test_value1\n",
+                    "test_user2,test_value2\n",
+                ]
+            )
         runner.invoke(
             cli,
             ["departing-employee", "bulk", "remove", "test_remove.csv"],
             obj=cli_state_with_user,
         )
-    assert bulk_processor.call_args[0][1] == ["test_user1", "test_user2"]
+    assert bulk_processor.call_args[0][1] == [
+        {"username": "test_user1"},
+        {"username": "test_user2"},
+    ]
+
+
+def test_remove_bulk_users_uses_expected_arguments_when_flat_file(
+    runner, mocker, cli_state_with_user
+):
+    bulk_processor = mocker.patch("code42cli.cmds.departing_employee.run_bulk_process")
+    with runner.isolated_filesystem():
+        with open("test_remove.txt", "w") as csv:
+            csv.writelines(["# username\n", "test_user1\n", "test_user2\n"])
+        runner.invoke(
+            cli,
+            ["departing-employee", "bulk", "remove", "test_remove.txt"],
+            obj=cli_state_with_user,
+        )
+    assert bulk_processor.call_args[0][1] == [
+        {"username": "test_user1"},
+        {"username": "test_user2"},
+    ]
 
 
 def test_add_departing_employee_when_invalid_date_validation_raises_error(
@@ -350,7 +413,7 @@ def test_remove_departing_employee_when_user_not_on_list_prints_expected_error(
         (f"{DEPARTING_EMPLOYEE_COMMAND} add", "Missing argument 'USERNAME'."),
         (f"{DEPARTING_EMPLOYEE_COMMAND} remove", "Missing argument 'USERNAME'.",),
         (f"{DEPARTING_EMPLOYEE_COMMAND} bulk add", "Missing argument 'CSV_FILE'.",),
-        (f"{DEPARTING_EMPLOYEE_COMMAND} bulk remove", "Missing argument 'FILE'.",),
+        (f"{DEPARTING_EMPLOYEE_COMMAND} bulk remove", "Missing argument 'CSV_FILE'.",),
     ],
 )
 def test_departing_employee_command_when_missing_required_parameters_returns_error(

tests/cmds/test_devices.py

@@ -942,6 +942,28 @@ def test_bulk_deactivate_uses_expected_arguments(runner, mocker, cli_state):
     ]
 
 
+def test_bulk_deactivate_uses_expected_arguments_when_no_header(
+    runner, mocker, cli_state
+):
+    bulk_processor = mocker.patch(f"{_NAMESPACE}.run_bulk_process")
+    with runner.isolated_filesystem():
+        with open("test_bulk_deactivate.csv", "w") as csv:
+            csv.writelines(["test_guid1\n"])
+        runner.invoke(
+            cli,
+            ["devices", "bulk", "deactivate", "test_bulk_deactivate.csv"],
+            obj=cli_state,
+        )
+    assert bulk_processor.call_args[0][1] == [
+        {
+            "guid": "test_guid1",
+            "deactivated": "False",
+            "change_device_name": False,
+            "purge_date": None,
+        }
+    ]
+
+
 def test_bulk_deactivate_ignores_blank_lines(runner, mocker, cli_state):
     bulk_processor = mocker.patch(f"{_NAMESPACE}.run_bulk_process")
     with runner.isolated_filesystem():
@@ -1001,6 +1023,23 @@ def test_bulk_reactivate_uses_expected_arguments(runner, mocker, cli_state):
     assert bulk_processor.call_args[0][1] == [{"guid": "test", "reactivated": "False"}]
 
 
+def test_bulk_reactivate_uses_expected_arguments_when_no_header(
+    runner, mocker, cli_state
+):
+    bulk_processor = mocker.patch(f"{_NAMESPACE}.run_bulk_process")
+    with runner.isolated_filesystem():
+        with open("test_bulk_reactivate.csv", "w") as csv:
+            csv.writelines(["test_guid1\n"])
+        runner.invoke(
+            cli,
+            ["devices", "bulk", "reactivate", "test_bulk_reactivate.csv"],
+            obj=cli_state,
+        )
+    assert bulk_processor.call_args[0][1] == [
+        {"guid": "test_guid1", "reactivated": "False"},
+    ]
+
+
 def test_bulk_reactivate_ignores_blank_lines(runner, mocker, cli_state):
     bulk_processor = mocker.patch(f"{_NAMESPACE}.run_bulk_process")
     with runner.isolated_filesystem():