Explicitly add allow_pickle=False (default) to all np.load() calls

Will change to True only for the calls that may possibly load trusted
Pickles, not only generic .npz files.

https://docs.scipy.org/doc/numpy/reference/generated/numpy.load.html
Changed in version 1.16.3: Made default False in response to CVE-2019-6446.

Reported by @Wouter-VDP via Slack on 2019-08-08.

Author: felker

examples/notebooks/Signal Influence.ipynb

@@ -26,20 +26,16 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "dat = np.load('./temp_data/signal_influence/signal_influence_results_155191_2017-12-01-01-42-33.npz')"
+    "dat = np.load('./temp_data/signal_influence/signal_influence_results_155191_2017-12-01-01-42-33.npz', allow_pickle=False)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 7,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "name": "stdout",
@@ -60,9 +56,7 @@
   {
    "cell_type": "code",
    "execution_count": 8,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "data": {
@@ -102,9 +96,7 @@
   {
    "cell_type": "code",
    "execution_count": 9,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "data": {
@@ -139,9 +131,7 @@
   {
    "cell_type": "code",
    "execution_count": 17,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "name": "stdout",
@@ -211,9 +201,7 @@
   {
    "cell_type": "code",
    "execution_count": 15,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "data": {
@@ -315,15 +303,13 @@
    },
    "outputs": [],
    "source": [
-    "dat = np.load('./temp_data/signal_influence/signal_influence_results_2017-11-30-20-13-39.npz')"
+    "dat = np.load('./temp_data/signal_influence/signal_influence_results_2017-11-30-20-13-39.npz', allow_pickle=False)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 4,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "y_gold = dat['y_gold']\n",
@@ -333,9 +319,7 @@
   {
    "cell_type": "code",
    "execution_count": 97,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "data": {
@@ -362,9 +346,7 @@
   {
    "cell_type": "code",
    "execution_count": 18,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "t_range = np.linspace(0,len(y_gold[0]),10,dtype=np.int)"
@@ -373,9 +355,7 @@
   {
    "cell_type": "code",
    "execution_count": 19,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "data": {
@@ -395,9 +375,7 @@
   {
    "cell_type": "code",
    "execution_count": 95,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "def difference_metric(y_prime,y_prime_orig):\n",
@@ -422,9 +400,7 @@
   {
    "cell_type": "code",
    "execution_count": 96,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "differences = get_importance_measure(y_prime,difference_metric)"
@@ -433,9 +409,7 @@
   {
    "cell_type": "code",
    "execution_count": 24,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "data": {
@@ -464,21 +438,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python [conda root]",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "conda-root-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
   },
   "latex_envs": {
    "LaTeX_envs_menu_present": true,

plasma/conf_parser.py

@@ -23,14 +23,15 @@ def parameters(input_file):
         base_path = output_path
 
         params['paths']['base_path'] = base_path
-        params['paths']['signal_prepath'] = base_path + \
-            params['paths']['signal_prepath']
-        params['paths']['shot_list_dir'] = base_path + \
-            params['paths']['shot_list_dir']
+        params['paths']['signal_prepath'] = (
+            base_path + params['paths']['signal_prepath'])
+        params['paths']['shot_list_dir'] = (
+            base_path + params['paths']['shot_list_dir'])
         params['paths']['output_path'] = output_path
         h = get_unique_signal_hash(sig.all_signals.values())
-        params['paths']['global_normalizer_path'] = output_path + \
-            '/normalization/normalization_signal_group_{}.npz'.format(h)
+        params['paths']['global_normalizer_path'] = (
+            output_path
+            + '/normalization/normalization_signal_group_{}.npz'.format(h))
         if params['training']['hyperparam_tuning']:
             # params['paths']['saved_shotlist_path'] =
             # './normalization/shot_lists.npz'
@@ -44,12 +45,12 @@ def parameters(input_file):
             # '/normalization/shot_lists.npz'
             params['paths']['normalizer_path'] = (
                 params['paths']['global_normalizer_path'])
-            params['paths']['model_save_path'] = (
-                output_path + '/model_checkpoints/')
+            params['paths']['model_save_path'] = (output_path
+                                                  + '/model_checkpoints/')
             params['paths']['csvlog_save_path'] = output_path + '/csv_logs/'
             params['paths']['results_prepath'] = output_path + '/results/'
-        params['paths']['tensorboard_save_path'] = output_path + \
-            params['paths']['tensorboard_save_path']
+        params['paths']['tensorboard_save_path'] = (
+            output_path + params['paths']['tensorboard_save_path'])
         params['paths']['saved_shotlist_path'] = (
             params['paths']['base_path'] + '/processed_shotlists/'
             + params['paths']['data']

plasma/models/mpi_runner.py

@@ -670,7 +670,7 @@ def save_shotlists(conf, shot_list_train, shot_list_validate, shot_list_test):
 
 def load_shotlists(conf):
     path = get_shot_list_path(conf)
-    data = np.load(path)
+    data = np.load(path, allow_pickle=False)
     shot_list_train = data['shot_list_train'][()]
     shot_list_validate = data['shot_list_validate'][()]
     shot_list_test = data['shot_list_test'][()]

plasma/models/shallow_runner.py

@@ -137,7 +137,7 @@ def process(self, shot):
             # print(X.shape, Y.shape)
         else:
             try:
-                dat = np.load(save_path)
+                dat = np.load(save_path, allow_pickle=False)
                 # X, Y, disr = dat["X"], dat["Y"], dat["disr"][()]
                 X = dat["X"]
             except BaseException:

plasma/preprocessor/normalize.py

@@ -185,7 +185,7 @@ def previously_saved_stats(self):
         if not os.path.isfile(self.path):
             return False, set([])
         else:
-            dat = np.load(self.path, encoding="latin1")
+            dat = np.load(self.path, encoding="latin1", allow_pickle=False)
             machines = dat['machines'][()]
             ret = all(
                 [m in machines for m in self.conf['paths']['all_machines']])
@@ -292,7 +292,7 @@ def save_stats(self):
 
     def load_stats(self):
         assert self.previously_saved_stats()[0], "stats not saved before"
-        dat = np.load(self.path, encoding="latin1")
+        dat = np.load(self.path, encoding="latin1", allow_pickle=False)
         self.means = dat['means'][()]
         self.stds = dat['stds'][()]
         self.num_processed = dat['num_processed'][()]
@@ -448,7 +448,7 @@ def save_stats(self):
 
     def load_stats(self):
         assert(self.previously_saved_stats()[0])
-        dat = np.load(self.path, encoding="latin1")
+        dat = np.load(self.path, encoding="latin1", allow_pickle=False)
         self.minimums = dat['minimums'][()]
         self.maximums = dat['maximums'][()]
         self.num_processed = dat['num_processed'][()]

plasma/preprocessor/preprocess.py

@@ -149,7 +149,7 @@ def get_shot_list_path(self):
 
     def load_shotlists(self):
         path = self.get_shot_list_path()
-        data = np.load(path, encoding="latin1")
+        data = np.load(path, encoding="latin1", allow_pickle=False)
         shot_list_train = data['shot_list_train'][()]
         shot_list_validate = data['shot_list_validate'][()]
         shot_list_test = data['shot_list_test'][()]
@@ -159,18 +159,13 @@ def load_shotlists(self):
             return ShotList(shot_list_train), ShotList(
                 shot_list_validate), ShotList(shot_list_test)
 
-    def save_shotlists(
-            self,
-            shot_list_train,
-            shot_list_validate,
-            shot_list_test):
+    def save_shotlists(self, shot_list_train, shot_list_validate,
+                       shot_list_test):
         path = self.get_shot_list_path()
         mkdirdepth(path)
-        np.savez(
-            path,
-            shot_list_train=shot_list_train,
-            shot_list_validate=shot_list_validate,
-            shot_list_test=shot_list_test)
+        np.savez(path, shot_list_train=shot_list_train,
+                 shot_list_validate=shot_list_validate,
+                 shot_list_test=shot_list_test)
 
 
 def apply_bleed_in(conf, shot_list_train, shot_list_validate, shot_list_test):

plasma/primitives/shots.py

@@ -498,7 +498,7 @@ def get_save_path(self, prepath):
     def restore(self, prepath, light=False):
         assert self.previously_saved(prepath), 'shot was never saved'
         save_path = self.get_save_path(prepath)
-        dat = np.load(save_path, encoding="latin1")
+        dat = np.load(save_path, encoding="latin1", allow_pickle=False)
 
         self.valid = dat['valid'][()]
         self.is_disruptive = dat['is_disruptive'][()]

plasma/utils/performance.py

@@ -364,7 +364,8 @@ def get_accuracy_and_fp_rate_from_stats(
     def load_ith_file(self):
         results_files = os.listdir(self.results_dir)
         print(results_files)
-        dat = np.load(self.results_dir + results_files[self.i])
+        dat = np.load(self.results_dir + results_files[self.i],
+                      allow_pickle=False)
         print("Loading results file {}".format(
             self.results_dir + results_files[self.i]))