1. add on_delete file_to_upload remove file from the upload list. 2. added filename prefix and suffic rules

Author: andrewluiqut

README.md

@@ -77,9 +77,10 @@ Determines whether the uploader provides a web interface (`web`) or no interface
 ```yaml
 # cloud name, as appeared in the clouds.yaml file
 uploader.filestore.cloud: openstack
-# filestore path
+# the local filestore path where new or modified files are uploaded
 uploader.filestore.local: /home/qcr/Bagfiles
-uploader.filestore.cloud.container: Bagfiles
+# the target object store container, the uploader will attempt to create if it not already exists
+uploader.filestore.cloud.container: Shorts Bagfiles      
 ```
 The cloud name specified in the `clouds.yaml` should be set against `uploader.filestore.cloud`. If Nectar or another openstack compliant cloud service is used, then the `clouds.yaml` should normally use `openstack`. The name is found under `clouds` in the yaml file.
 ```yaml
@@ -93,6 +94,15 @@ The `uploader.filestore.local` specifies the local directory where new files are
 
 The `uploader.filestore.cloud.container` specifies the container name in the object store where the files are uploaded. If the container does not exist, the uploader will create it.
 
+```yaml
+# exclude files with suffixes and prefixes
+uploader.ignore.suffix:
+  - .active
+uploader.ignore.prefix:
+  - '~'
+```
+The two yaml keys, `uploader.ignore.suffix` and `uploader.ignore.prefix`, specify a list of suffixes and prefixes of filenames that will be ignored. To define more than one suffix or prefix, add more lines under the key as a list. 
+
 ```yaml
 uploader.web.host: 0.0.0.0
 uploader.web.port: 8071

config/uploader_config.yaml

@@ -4,18 +4,22 @@
 uploader.mode: web # web or headless
 # cloud name, as appeared in the clouds.yaml file
 uploader.filestore.cloud: openstack
-# filestore path
-uploader.filestore.local: /home/qcr/Insync/Bagfiles
-uploader.filestore.cloud.container: Shorts Bagfiles
+# the local filestore path where new or modified files are uploaded
+uploader.filestore.local: /home/qcr/Bagfiles
+# the target object store container, the uploader will attempt to create if it not already exists
+uploader.filestore.cloud.container: Shorts Bagfiles      
+# exclude files with suffixes and prefixes
+uploader.ignore.suffix:
+  - .active
+uploader.ignore.prefix:
+  - '~'
 # Upload delay
 uploader.delay: 30 # seconds
 uploader.error_count.max: 5 
 # Dash Agent (uploader) setting
 uploader.web.host: 0.0.0.0
 uploader.web.port: 8071
-# uploader.web.debug.mode: False
 uploader.web.launch_browser: False
-uploader.web.auth: True   # Authentication required if web mode
 # Main system timers for the uploader implemented by Dash
 uploader.system.timer: 1  # seconds
 uploader.console.refresh: 5  # seconds

launch/main.launch

@@ -4,7 +4,7 @@
         <!-- Launch the camera monitor node first, which will launch the camera agent -->
         <group if="$(arg run_uploader)"> 
             <param name="mode" type="string" value="$(arg mode)" /> 
-            <node pkg="openstack_object_uploader" type="run.py" name="openstack_object_uploader_node" output="screen" respawn="true" >
+            <node pkg="openstack_object_uploader" type="run.py" name="openstack_object_uploader_node" output="screen" respawn="false" >
             </node>
         </group>
 </launch>   

src/uploader/run.py

@@ -34,6 +34,25 @@ def __init__(self, filestore_path):
         super(CustomFileSystemEventHandler, self,).__init__()
         self.filestore_path = filestore_path
         self.uploader_delay = CONFIG.get('uploader.delay', 30)
+        # setup ignore list
+        self.ignore_suffix = CONFIG.get('uploader.ignore.suffix', None)
+        if self.ignore_suffix is not None and type(self.ignore_suffix) == str:
+            self.ignore_suffix = [self.ignore_suffix]
+        self.ignore_prefix = CONFIG.get('uploader.ignore.prefix', None)
+        if self.ignore_prefix is not None and type(self.ignore_prefix) == str:
+            self.ignore_prefix = [self.ignore_prefix]  
+
+    def is_in_ignore_lists(self, filename:str) -> bool:
+        if self.ignore_suffix is not None and type(self.ignore_suffix) in (tuple, list):
+            for prefix in self.ignore_suffix:
+                if filename.endswith(prefix):
+                    return True
+        if self.ignore_prefix is not None and type(self.ignore_prefix) in (tuple, list):
+            for prefix in self.ignore_prefix:
+                if filename.startswith(prefix):
+                    return True        
+        return False
+
     # The callback function when a create event occurs
     def on_created(self, event:FileSystemEvent):
         pass
@@ -43,10 +62,18 @@ def on_modified(self, event:FileSystemEvent):
             # if the event is from a modified file
             local_path = event.src_path
             filename = file_tools.get_filename(local_path)
+            if self.is_in_ignore_lists(filename):
+                return
             parent_path = file_tools.get_parent(local_path)
             sub_path = parent_path[len(self.filestore_path) + 1:]  # the sub_path cannot start with '/' for os.path.join to work
             remote_filename = os.path.join(sub_path, filename)
             DAO.add_upload_file(local_path, filename, remote_filename, int(time.time()) + CONFIG.get('uploader.delay', self.uploader_delay))
+    # the callback function when a file is deleted
+    def on_deleted(self, event:FileSystemEvent):
+        if not event.is_directory:
+            # if the event is from a file
+            local_path = event.src_path
+            DAO.remove_upload_file(local_path)
 
 class OpenstackObjectUploader(object):
     def __init__(self):
@@ -63,6 +90,7 @@ def __init__(self):
         try:
             self.cloud = openstack.connect(cloud=self.cloud_name)
         except openstack.exceptions.ConfigException as e:
+            self.cloud = None
             logger.error(f'{type(self).__name__}: The application is unable to locate the application credential file "clouds.yaml".')
             logger.warning(f'{type(self).__name__}: Ensure that a clouds.yaml file is obtained from the openstack cloud provider.')
             logger.warning(f'{type(self).__name__}: If you have a valid clouds.yaml file, make sure it is in the current directory or ~/.config/openstack and only one cloud.yaml file.')
@@ -93,14 +121,15 @@ def __init__(self):
             sys.exit(1)   
         # attempt to make a connection to the container
         try:
-            self.container = self.cloud.object_store.create_container(self.filestore_container, is_content_type_detected=True)
-            logger.info(f'{type(self).__name__}: Connected to (and if needed created) the container "{self.filestore_container}"')
+            if self.cloud is not None:
+                self.container = self.cloud.object_store.create_container(self.filestore_container, is_content_type_detected=True)
+                logger.info(f'{type(self).__name__}: Connected to (and if needed created) the container "{self.filestore_container}"')
         except keystoneauth1.exceptions.http.Unauthorized as e:
             logger.error(f'{type(self).__name__}: The application is unable to authenticate using the credentials in "clouds.yaml".')
             logger.warning(f'{type(self).__name__}: Ensure that the file is current and valid to the cloud server.')
             logger.warning(f'{type(self).__name__}: The access right should include granting access to an object store.')
             logger.warning(f'{type(self).__name__}: Terminate the application and fix the problem according to the README.md file.')
-            sys.exit(1)
+            sys.exit(1)          
 
         # start the watchdog 
         self.watchdog_thread = self.run_watchdog(self.filestore_local)
@@ -174,14 +203,14 @@ def run_uploader(self):
             if len(next_to_upload) == 0:
                 continue
             # starts the uploading procedure
-            model.STATE.update(model.SystemStates.UPLOADING)
             # compute the parameters for uploading including the local and remote path
             model.STATE.set_var('upload', next_to_upload[0])
             local_path = next_to_upload[0]['local_path']
             remote_path = next_to_upload[0]['remote_path']
             file_size = (os.stat(local_path).st_size)
             start_time = time.time()
             logger.info(f'{type(self).__name__} (run_uploader): Uploading file {local_path} to {remote_path}')
+            model.STATE.update(model.SystemStates.UPLOADING)
             try: 
                 self.cloud.object_store.upload_object(container=self.container, name=remote_path, filename=local_path)
                 # compute the upload time

src/uploader/web/dashapp_top.py

@@ -64,7 +64,7 @@ def _define_app(self):
         self._navbar_with_menu = dbc.NavbarSimple(
                     children=[
                         dbc.NavItem(dbc.NavLink('Console', href='/page_console')),  
-                        dbc.NavItem(dbc.NavLink('DB', href='/page_db_browser')),                                              
+                        # dbc.NavItem(dbc.NavLink('DB', href='/page_db_browser')),                                              
                     ],
                     brand=html.Div([html.H3(_APP_NAME), html.H6('Robotics and Autonomous Systems Group, REF, RI, Queensland University of Technology')]),
                     brand_href='/page_console', color='#ffc6aa', className='fs-4 text')
@@ -108,8 +108,8 @@ def display_page(pathname):
             try: 
                 if pathname == '/page_console':
                     page_content = self._console_page.layout()          
-                elif pathname == '/page_db_browser':
-                    page_content = self._db_browse_page.layout()                                        
+                # elif pathname == '/page_db_browser':
+                #     page_content = self._db_browse_page.layout()                                        
                 elif pathname == '/': 
                     page_content = self._console_page.layout()
                 else: # if redirected to unknown link