overwrite merging in branch, keeping changes to scripts, spec and tests

Author: Clare72

.gitignore

@@ -1,3 +1,5 @@
+.DS_Store
 venv
 .idea
-.DS_Store
+.DS_Store
+*.pyc

records/new_images/common_fields_spec.yaml

@@ -89,7 +89,7 @@ dbxrefs:
   description: >
      Cross references for image/depicted entity in other databases taking the form db:accession where db
      is the short_form of a Site node in VFB and accession is an identifier in a third party database.
-  uniq: True
+  uniq: False
   type: String
   multiple: True
   pattern_arg: dbxref_strings

records/new_images/split_spec.yaml

@@ -36,3 +36,9 @@ DBD:
   uniq: False
   flybase_feature: true
   multiple: False
+  
+is_a:
+  compulsory: False
+  uniq: False
+  multiple: True
+

records/new_metadata/newmeta_spec.yaml

@@ -121,5 +121,3 @@ weight_per_roi:
 
 
 
-
-

records/new_splits/relations_spec.yaml

@@ -0,0 +1 @@
+# Dummy relations spec file to hack split addition

records/new_splits/split_spec.yaml

@@ -0,0 +1,56 @@
+AD_id:
+  compulsory: False
+  description: >
+     The FBid of the feature corresponding to the activiation domain half of the split in FlyBase.
+     Please use transposon rather than insertion ID.
+  uniq: False
+  type: Feature
+  multiple: False
+
+AD:
+  compulsory: True
+  description: >
+     The name of the feature corresponding to the activiation domain half of the split in FlyBase.
+     Please use transposon rather than insertion.
+  uniq: False
+  type: Feature
+  multiple: False
+  flybase_feature: true
+
+
+DBD_id:
+  compulsory: False
+  description: >
+     The FBid of the feature corresponding to the DNA binding domain half of the split in FlyBase.
+     Please use transposon rather than insertion ID.
+  uniq: False
+  type: Feature
+  multiple: False
+
+DBD:
+  compulsory: True
+  description: >
+     The name of the feature corresponding to the DNA binding domain half
+     of the split in FlyBase. Please use transposon rather than insertion.
+  uniq: False
+  flybase_feature: true
+  multiple: False
+
+synonyms:
+  compulsory: False
+  description: One or more additional names used to refer to the entity.
+  uniq: False
+  type: String
+  multiple: True
+  pattern_arg: anatomy_attributes
+  pattern_dict_key: synonyms
+
+dbxrefs:
+  compulsory: False
+  description: >
+     Cross references for image/depicted entity in other databases taking the form db:accession where db
+     is the short_form of a Site node in VFB and accession is an identifier in a third party database.
+  uniq: True
+  type: String
+  multiple: True
+  pattern_arg: dbxref_strings

records/relations_spec.yaml

@@ -13,7 +13,7 @@ part_of:
   type: objectProperty
   restriction:
     field: short_form
-    regex: "(FBbt|VFB)_.+"
+    regex: "(FBbt|VFB|VFBexp)_.+"
     labels: ['Class']
 
 expresses:

src/run_peevish.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 
+import logging
 from vfb.curation.peevish import get_recs
 from vfb.curation.curation_loader import load_recs
-#from vfb.curation.cur_load import NewMetaDataWriter, NewImageWriter
 import argparse
 import warnings
 import os
@@ -13,60 +13,104 @@
 2. Test load records in 'to_submit/'.
 """
 # What logic, if any should live here?
-
 ## Should it know about the gross type of records and direct to the appropriate loading method, where gross currently = new_dataSet vs new_metadata, vs new image.  Shouldn't a record sorter be able to do that?
-
-
+ 
+# Argument parser setup
 parser = argparse.ArgumentParser()
-parser.add_argument("endpoint",
-                    help="Endpoint for connection to neo4J prod")
-parser.add_argument("usr",
-                    help="username")
-parser.add_argument("pwd",
-                    help="password")
-parser.add_argument("--import_filepath",
-                    help="path to file usable for writing to DB via tsv", default='')
+parser.add_argument("endpoint", help="Endpoint for connection to neo4J prod")
+parser.add_argument("usr", help="username")
+parser.add_argument("pwd", help="password")
+parser.add_argument("--import_filepath", help="path to file usable for writing to DB via tsv", default='')
 parser.add_argument("--base_path", help="Optional", default="../")
 parser.add_argument("--test_mode", help="Optional", action='store_true', default=False)
-parser.add_argument("--commit",  help="Optional", action='store_true', default=False)
-parser.add_argument("--verbose",  help="Optional", action='store_true', default=False)
-parser.add_argument("--allow_duplicates",  help="Optional", action='store_true', default=False)
+parser.add_argument("--commit", help="Optional", action='store_true', default=False)
+parser.add_argument("--verbose", help="Optional", action='store_true', default=False)
+parser.add_argument("--allow_duplicates", help="Optional", action='store_true', default=False)
+parser.add_argument("--debug", help="Enable debug logging", action='store_true', default=False)
 args = parser.parse_args()
 
-
-
-def check_records(path, check_dir = "working"):
-    rec_path = '/'.join([args.base_path, path, check_dir]) + '/'
-    recs = get_recs(spec_path="../records/" + path,
-                    path_to_recs=rec_path)
-    stat = True
-    if len(recs) == 0:
-        print("No records to check in: " + rec_path)
-    else:
-        print("Testing syntax of %s curation records." % len(recs))
-    if False in recs:
-        stat = False
-    return stat
-
-
-
-def load_records(path, load_dir = "to_submit"):
-    rec_path = os.path.abspath(os.path.join(args.base_path, path, load_dir))
-    current_dir = os.path.dirname(os.path.realpath(__file__))
-    records_path = os.path.abspath(os.path.join(current_dir, "../records", path))
-    stat = load_recs(records_path + '/', rec_path + '/',
-                     args.endpoint, args.usr, args.pwd, import_filepath=args.import_filepath,
-                     commit=args.commit, verbose=args.verbose, allow_duplicates=args.allow_duplicates)
-    return stat
-
+# Configure logging
+if args.debug:
+    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
+elif args.verbose:
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+else:
+    logging.basicConfig(level=logging.WARNING, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def check_records(path, check_dir="working"):
+    """
+    Checks the records in the specified directory.
+    
+    Parameters:
+    path (str): The base path to check records in.
+    check_dir (str): The directory within the base path to check records in.
+    
+    Returns:
+    bool: True if records pass the check, False otherwise.
+    """
+    try:
+        logging.debug("Starting check_records with path: %s and check_dir: %s", path, check_dir)
+        rec_path = '/'.join([args.base_path, path, check_dir]) + '/'
+        logging.debug("Constructed rec_path: %s", rec_path)
+        
+        recs = get_recs(spec_path="../records/" + path, path_to_recs=rec_path)
+        logging.debug("Retrieved records: %s", recs)
+        
+        stat = True
+        if len(recs) == 0:
+            logging.info("No records to check in: %s", rec_path)
+        else:
+            logging.info("Testing syntax of %s curation records.", len(recs))
+        
+        if False in recs:
+            logging.warning("Some records failed the check.")
+            stat = False
+        
+        return stat
+
+    except Exception as e:
+        logging.error("An error occurred in check_records: %s", e, exc_info=True)
+        return False
+
+def load_records(path, load_dir="to_submit"):
+    """
+    Loads the records from the specified directory.
+    
+    Parameters:
+    path (str): The base path to load records from.
+    load_dir (str): The directory within the base path to load records from.
+    
+    Returns:
+    bool: True if records are successfully loaded, False otherwise.
+    """
+    try:
+        logging.debug("Starting load_records with path: %s and load_dir: %s", path, load_dir)
+        rec_path = os.path.abspath(os.path.join(args.base_path, path, load_dir))
+        logging.debug("Constructed rec_path: %s", rec_path)
+        
+        current_dir = os.path.dirname(os.path.realpath(__file__))
+        records_path = os.path.abspath(os.path.join(current_dir, "../records", path))
+        logging.debug("Constructed records_path: %s", records_path)
+        
+        stat = load_recs(records_path + '/', rec_path + '/',
+                         args.endpoint, args.usr, args.pwd, import_filepath=args.import_filepath,
+                         commit=args.commit, verbose=args.verbose, allow_duplicates=args.allow_duplicates)
+        
+        logging.debug("load_recs returned status: %s", stat)
+        return stat
+
+    except Exception as e:
+        logging.error("An error occurred in load_records: %s", e, exc_info=True)
+        return False
 
 stat = True
 
-
 if not check_records(path="new_metadata/"): stat = False
 if not load_records(path="new_metadata/"): stat = False
 if not check_records(path="new_images/"): stat = False
 if not load_records(path="new_images/"): stat = False
+if not check_records(path="new_splits/"): stat = False
+if not load_records(path="new_splits/"): stat = False
 
 if not stat:
     raise Exception("Failing records. See preceding warnings for details.")
@@ -77,8 +121,9 @@ def load_records(path, load_dir = "to_submit"):
     print("Running record syntax fails tests")
     check_records(path="new_metadata/", check_dir="test_syntax_fail")
     check_records(path="new_images/", check_dir="test_syntax_fail")
+    # check_records(path="new_splits/", check_dir="test_syntax_fail")
+
     print("Running fail tests.")
     load_records(path="new_metadata/", load_dir="test_load_fail")
     load_records(path="new_images/", load_dir="test_load_fail")
-
-
+    # load_records(path="new_splits/", load_dir="test_load_fail")