Module Gnumed.pycommon.gmBackendListener

GNUmed database backend listener.

This module implements threaded listening for asynchronuous notifications from the database backend.

Expand source code
"""GNUmed database backend listener.

This module implements threaded listening for asynchronuous
notifications from the database backend.
"""
#=====================================================================
__author__ = "H. Herb <hherb@gnumed.net>, K.Hilbert <karsten.hilbert@gmx.net>"
__license__ = "GPL v2 or later"

import sys
import time
import threading
import select
import logging


if __name__ == '__main__':
        sys.path.insert(0, '../../')
from Gnumed.pycommon import gmDispatcher
from Gnumed.pycommon import gmBorg


_log = logging.getLogger('gm.db')


signals2listen4 = [
        'db_maintenance_warning',               # warns of impending maintenance and asks for disconnect
        'db_maintenance_disconnect',    # announces a forced disconnect and disconnects
        'gm_table_mod'                                  # sent for any (registered) table modification, payload contains details
]

#=====================================================================
class gmBackendListener(gmBorg.cBorg):
        """The backend listener singleton class."""
        def __init__(self, conn=None, poll_interval=3):
                try:
                        # pylint: disable-next=access-member-before-definition
                        self.already_inited
                        return

                except AttributeError:
                        pass

                self.debug = False
                self.__notifications_received = 0
                self.__messages_sent = 0

                _log.info('starting backend notifications listener thread')

                # the listener thread will regularly try to acquire
                # this lock, when it succeeds it will quit
                self._quit_lock = threading.Lock()
                # take the lock now so it cannot be taken by the worker
                # thread until it is released in shutdown()
                if not self._quit_lock.acquire(0):
                        _log.error('cannot acquire thread-quit lock, aborting')
                        raise EnvironmentError("cannot acquire thread-quit lock")

                self._conn = conn
                _log.debug('DB listener connection: %s', self._conn)
                self.backend_pid = self._conn.get_backend_pid()
                _log.debug('notification listener connection has backend PID [%s]', self.backend_pid)
                self._conn.set_isolation_level(0)               # autocommit mode = psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT
                self._cursor = self._conn.cursor()
                try:
                        self._conn_fd = self._conn.fileno()
                except AttributeError:
                        self._conn_fd = self._cursor.fileno()
                self._conn_lock = threading.Lock()              # lock for access to connection object

                self.__register_interests()

                # check for messages every 'poll_interval' seconds
                self._poll_interval = poll_interval
                self._listener_thread = None
                self.__start_thread()

                self.already_inited = True

        #-------------------------------
        # public API
        #-------------------------------
        def shutdown(self):
                """Cleanly shut down listening.

                Unregister notifications. Rejoin listener thread.
                """
                _log.debug('received %s notifications', self.__notifications_received)
                _log.debug('sent %s messages', self.__messages_sent)
                if self._listener_thread is None:
                        self.__shutdown_connection()
                        return

                _log.info('stopping backend notifications listener thread')
                self._quit_lock.release()
                try:
                        # give the worker thread time to terminate
                        self._listener_thread.join(self._poll_interval+2.0)
                        try:
                                if self._listener_thread.is_alive():
                                        _log.error('listener thread still alive after join()')
                                        _log.debug('active threads: %s' % threading.enumerate())
                        except Exception:
                                pass
                except Exception:
                        print(sys.exc_info())
                self._listener_thread = None
                try:
                        self.__unregister_unspecific_notifications()
                except Exception:
                        _log.exception('unable to unregister unspecific notifications')

                self.__shutdown_connection()
                return

        #-------------------------------
        # event handlers
        #-------------------------------
        # internal helpers
        #-------------------------------
        def __register_interests(self):
                # determine unspecific notifications
                self.unspecific_notifications = signals2listen4
                _log.info('configured unspecific notifications:')
                _log.info('%s' % self.unspecific_notifications)
                gmDispatcher.known_signals.extend(self.unspecific_notifications)

                # listen to unspecific notifications
                self.__register_unspecific_notifications()

        #-------------------------------
        def __register_unspecific_notifications(self):
                for sig in self.unspecific_notifications:
                        _log.info('starting to listen for [%s]' % sig)
                        cmd = 'LISTEN "%s"' % sig
                        self._conn_lock.acquire(1)
                        try:
                                self._cursor.execute(cmd)
                        finally:
                                self._conn_lock.release()

        #-------------------------------
        def __unregister_unspecific_notifications(self):
                for sig in self.unspecific_notifications:
                        _log.info('stopping to listen for [%s]' % sig)
                        cmd = 'UNLISTEN "%s"' % sig
                        self._conn_lock.acquire(1)
                        try:
                                self._cursor.execute(cmd)
                        finally:
                                self._conn_lock.release()

        #-------------------------------
        def __shutdown_connection(self):
                _log.debug('shutting down connection with backend PID [%s]', self.backend_pid)
                self._conn_lock.acquire(1)
                try:
                        self._conn.rollback()
                except Exception:
                        pass
                finally:
                        self._conn_lock.release()

        #-------------------------------
        def __start_thread(self):
                if self._conn is None:
                        raise ValueError("no connection to backend available, useless to start thread")

                self._listener_thread = threading.Thread (
                        target = self._process_notifications,
                        name = self.__class__.__name__,
                        daemon = True
                )
                _log.info('starting listener thread')
                self._listener_thread.start()

        #-------------------------------
        # the actual thread code
        #-------------------------------
        def _process_notifications(self):

                # loop until quitting
                _have_quit_lock = None
                while not _have_quit_lock:
                        # quitting ?
                        if self._quit_lock.acquire(0):
                                break

                        # wait at most self._poll_interval for new data
                        self._conn_lock.acquire(1)
                        try:
                                ready_input_sockets = select.select([self._conn_fd], [], [], self._poll_interval)[0]
                        finally:
                                self._conn_lock.release()
                        # any input available ?
                        if len(ready_input_sockets) == 0:
                                # no, select.select() timed out
                                # give others a chance to grab the conn lock (eg listen/unlisten)
                                time.sleep(0.3)
                                continue
                        # data available, wait for it to fully arrive
                        self._conn_lock.acquire(1)
                        try:
                                self._conn.poll()
                        finally:
                                self._conn_lock.release()
                        # any notifications ?
                        while len(self._conn.notifies) > 0:
                                # if self._quit_lock can be acquired we may be in
                                # __del__ in which case gmDispatcher is not
                                # guaranteed to exist anymore
                                if self._quit_lock.acquire(0):
                                        _have_quit_lock = 1
                                        break

                                self._conn_lock.acquire(1)
                                try:
                                        notification = self._conn.notifies.pop()
                                finally:
                                        self._conn_lock.release()
                                self.__notifications_received += 1
                                if self.debug:
                                        print(notification)
                                _log.debug('#%s: %s (first param: PID of sending backend; this backend: %s)', self.__notifications_received, notification, self.backend_pid)
                                # decode payload
                                payload = notification.payload.split('::')
                                operation = None
                                table = None
                                pk_column_name = None
                                pk_of_row = None
                                pk_identity = None
                                for item in payload:
                                        if item.startswith('operation='):
                                                operation = item.split('=')[1]
                                        if item.startswith('table='):
                                                table = item.split('=')[1]
                                        if item.startswith('PK name='):
                                                pk_column_name = item.split('=')[1]
                                        if item.startswith('row PK='):
                                                pk_of_row = int(item.split('=')[1])
                                        if item.startswith('person PK='):
                                                try:
                                                        pk_identity = int(item.split('=')[1])
                                                except ValueError:
                                                        _log.exception('error in change notification trigger')
                                                        pk_identity = -1
                                # try sending intra-client signals:
                                # 1) generic signal
                                self.__messages_sent += 1
                                try:
                                        gmDispatcher.send (
                                                signal = notification.channel,
                                                originated_in_database = True,
                                                listener_pid = self.backend_pid,
                                                sending_backend_pid = notification.pid,
                                                pk_identity = pk_identity,
                                                operation = operation,
                                                table = table,
                                                pk_column_name = pk_column_name,
                                                pk_of_row = pk_of_row,
                                                message_index = self.__messages_sent,
                                                notification_index = self.__notifications_received
                                        )
                                except Exception:
                                        print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (notification.channel, notification.pid))
                                        print(sys.exc_info())
                                # 2) dynamically emulated old style table specific signals
                                if table is not None:
                                        self.__messages_sent += 1
                                        signal = '%s_mod_db' % table
                                        _log.debug('emulating old-style table specific signal [%s]', signal)
                                        try:
                                                gmDispatcher.send (
                                                        signal = signal,
                                                        originated_in_database = True,
                                                        listener_pid = self.backend_pid,
                                                        sending_backend_pid = notification.pid,
                                                        pk_identity = pk_identity,
                                                        operation = operation,
                                                        table = table,
                                                        pk_column_name = pk_column_name,
                                                        pk_of_row = pk_of_row,
                                                        message_index = self.__messages_sent,
                                                        notification_index = self.__notifications_received
                                                )
                                        except Exception:
                                                print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (signal, notification.pid))
                                                print(sys.exc_info())

                                # there *may* be more pending notifications but
                                # we don't care when quitting
                                if self._quit_lock.acquire(0):
                                        _have_quit_lock = 1
                                        break

                # exit thread activity
                return

#=====================================================================
# main
#=====================================================================
if __name__ == "__main__":

        if len(sys.argv) < 2:
                sys.exit()

        if sys.argv[1] not in ['test', 'monitor']:
                sys.exit()


        notifies = 0

        from Gnumed.pycommon import gmPG2, gmI18N
        from Gnumed.business import gmPerson, gmPersonSearch

        gmI18N.activate_locale()
        gmI18N.install_domain(domain='gnumed')
        #-------------------------------
        def run_test():

                #-------------------------------
                def dummy(n):
                        return float(n)*n/float(1+n)
                #-------------------------------
                def OnPatientModified():
                        global notifies
                        notifies += 1
                        sys.stdout.flush()
                        print("\nBackend says: patient data has been modified (%s. notification)" % notifies)
                #-------------------------------
                try:
                        n = int(sys.argv[2])
                except Exception:
                        print("You can set the number of iterations\nwith the second command line argument")
                        n = 100000

                # try loop without backend listener
                print("Looping", n, "times through dummy function")
                i = 0
                t1 = time.time()
                while i < n:
                        dummy(i)
                        i += 1
                t2 = time.time()
                t_nothreads = t2-t1
                print("Without backend thread, it took", t_nothreads, "seconds")

                listener = gmBackendListener(conn = gmPG2.get_raw_connection())

                # now try with listener to measure impact
                print("Now in a new shell connect psql to the")
                print("database <gnumed_v9> on localhost, return")
                print("here and hit <enter> to continue.")
                input('hit <enter> when done starting psql')
                print("You now have about 30 seconds to go")
                print("to the psql shell and type")
                print(" notify patient_changed<enter>")
                print("several times.")
                print("This should trigger our backend listening callback.")
                print("You can also try to stop the demo with Ctrl-C !")

                #listener.register_callback('patient_changed', OnPatientModified)

                try:
                        counter = 0
                        while counter < 20:
                                counter += 1
                                time.sleep(1)
                                sys.stdout.flush()
                                print('.')
                        print("Looping",n,"times through dummy function")
                        i = 0
                        t1 = time.time()
                        while i < n:
                                dummy(i)
                                i += 1
                        t2 = time.time()
                        t_threaded = t2-t1
                        print("With backend thread, it took", t_threaded, "seconds")
                        print("Difference:", t_threaded-t_nothreads)
                except KeyboardInterrupt:
                        print("cancelled by user")

                listener.shutdown()

        #-------------------------------
        def run_monitor():

                print("starting up backend notifications monitor")

                def monitoring_callback(*args, **kwargs):
                        try:
                                kwargs['originated_in_database']
                                print('==> got notification from database "%s":' % kwargs['signal'])
                        except KeyError:
                                print('==> received signal from client: "%s"' % kwargs['signal'])
                        del kwargs['signal']
                        for key in kwargs:
                                print('    [%s]: %s' % (key, kwargs[key]))

                gmDispatcher.connect(receiver = monitoring_callback)

                listener = gmBackendListener(conn = gmPG2.get_raw_connection())
                print("listening for the following notifications:")
                print("1) unspecific:")
                for sig in listener.unspecific_notifications:
                        print('   - %s' % sig)

                while True:
                        pat = gmPersonSearch.ask_for_patient()
                        if pat is None:
                                break
                        print("found patient", pat)
                        gmPerson.set_active_patient(patient=pat)
                        print("now waiting for notifications, hit <ENTER> to select another patient")
                        input()

                print("cleanup")
                listener.shutdown()

                print("shutting down backend notifications monitor")

        #-------------------------------
        if sys.argv[1] == 'monitor':
                run_monitor()
        else:
                run_test()

#=====================================================================

Classes

class gmBackendListener (conn=None, poll_interval=3)

The backend listener singleton class.

Expand source code
class gmBackendListener(gmBorg.cBorg):
        """The backend listener singleton class."""
        def __init__(self, conn=None, poll_interval=3):
                try:
                        # pylint: disable-next=access-member-before-definition
                        self.already_inited
                        return

                except AttributeError:
                        pass

                self.debug = False
                self.__notifications_received = 0
                self.__messages_sent = 0

                _log.info('starting backend notifications listener thread')

                # the listener thread will regularly try to acquire
                # this lock, when it succeeds it will quit
                self._quit_lock = threading.Lock()
                # take the lock now so it cannot be taken by the worker
                # thread until it is released in shutdown()
                if not self._quit_lock.acquire(0):
                        _log.error('cannot acquire thread-quit lock, aborting')
                        raise EnvironmentError("cannot acquire thread-quit lock")

                self._conn = conn
                _log.debug('DB listener connection: %s', self._conn)
                self.backend_pid = self._conn.get_backend_pid()
                _log.debug('notification listener connection has backend PID [%s]', self.backend_pid)
                self._conn.set_isolation_level(0)               # autocommit mode = psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT
                self._cursor = self._conn.cursor()
                try:
                        self._conn_fd = self._conn.fileno()
                except AttributeError:
                        self._conn_fd = self._cursor.fileno()
                self._conn_lock = threading.Lock()              # lock for access to connection object

                self.__register_interests()

                # check for messages every 'poll_interval' seconds
                self._poll_interval = poll_interval
                self._listener_thread = None
                self.__start_thread()

                self.already_inited = True

        #-------------------------------
        # public API
        #-------------------------------
        def shutdown(self):
                """Cleanly shut down listening.

                Unregister notifications. Rejoin listener thread.
                """
                _log.debug('received %s notifications', self.__notifications_received)
                _log.debug('sent %s messages', self.__messages_sent)
                if self._listener_thread is None:
                        self.__shutdown_connection()
                        return

                _log.info('stopping backend notifications listener thread')
                self._quit_lock.release()
                try:
                        # give the worker thread time to terminate
                        self._listener_thread.join(self._poll_interval+2.0)
                        try:
                                if self._listener_thread.is_alive():
                                        _log.error('listener thread still alive after join()')
                                        _log.debug('active threads: %s' % threading.enumerate())
                        except Exception:
                                pass
                except Exception:
                        print(sys.exc_info())
                self._listener_thread = None
                try:
                        self.__unregister_unspecific_notifications()
                except Exception:
                        _log.exception('unable to unregister unspecific notifications')

                self.__shutdown_connection()
                return

        #-------------------------------
        # event handlers
        #-------------------------------
        # internal helpers
        #-------------------------------
        def __register_interests(self):
                # determine unspecific notifications
                self.unspecific_notifications = signals2listen4
                _log.info('configured unspecific notifications:')
                _log.info('%s' % self.unspecific_notifications)
                gmDispatcher.known_signals.extend(self.unspecific_notifications)

                # listen to unspecific notifications
                self.__register_unspecific_notifications()

        #-------------------------------
        def __register_unspecific_notifications(self):
                for sig in self.unspecific_notifications:
                        _log.info('starting to listen for [%s]' % sig)
                        cmd = 'LISTEN "%s"' % sig
                        self._conn_lock.acquire(1)
                        try:
                                self._cursor.execute(cmd)
                        finally:
                                self._conn_lock.release()

        #-------------------------------
        def __unregister_unspecific_notifications(self):
                for sig in self.unspecific_notifications:
                        _log.info('stopping to listen for [%s]' % sig)
                        cmd = 'UNLISTEN "%s"' % sig
                        self._conn_lock.acquire(1)
                        try:
                                self._cursor.execute(cmd)
                        finally:
                                self._conn_lock.release()

        #-------------------------------
        def __shutdown_connection(self):
                _log.debug('shutting down connection with backend PID [%s]', self.backend_pid)
                self._conn_lock.acquire(1)
                try:
                        self._conn.rollback()
                except Exception:
                        pass
                finally:
                        self._conn_lock.release()

        #-------------------------------
        def __start_thread(self):
                if self._conn is None:
                        raise ValueError("no connection to backend available, useless to start thread")

                self._listener_thread = threading.Thread (
                        target = self._process_notifications,
                        name = self.__class__.__name__,
                        daemon = True
                )
                _log.info('starting listener thread')
                self._listener_thread.start()

        #-------------------------------
        # the actual thread code
        #-------------------------------
        def _process_notifications(self):

                # loop until quitting
                _have_quit_lock = None
                while not _have_quit_lock:
                        # quitting ?
                        if self._quit_lock.acquire(0):
                                break

                        # wait at most self._poll_interval for new data
                        self._conn_lock.acquire(1)
                        try:
                                ready_input_sockets = select.select([self._conn_fd], [], [], self._poll_interval)[0]
                        finally:
                                self._conn_lock.release()
                        # any input available ?
                        if len(ready_input_sockets) == 0:
                                # no, select.select() timed out
                                # give others a chance to grab the conn lock (eg listen/unlisten)
                                time.sleep(0.3)
                                continue
                        # data available, wait for it to fully arrive
                        self._conn_lock.acquire(1)
                        try:
                                self._conn.poll()
                        finally:
                                self._conn_lock.release()
                        # any notifications ?
                        while len(self._conn.notifies) > 0:
                                # if self._quit_lock can be acquired we may be in
                                # __del__ in which case gmDispatcher is not
                                # guaranteed to exist anymore
                                if self._quit_lock.acquire(0):
                                        _have_quit_lock = 1
                                        break

                                self._conn_lock.acquire(1)
                                try:
                                        notification = self._conn.notifies.pop()
                                finally:
                                        self._conn_lock.release()
                                self.__notifications_received += 1
                                if self.debug:
                                        print(notification)
                                _log.debug('#%s: %s (first param: PID of sending backend; this backend: %s)', self.__notifications_received, notification, self.backend_pid)
                                # decode payload
                                payload = notification.payload.split('::')
                                operation = None
                                table = None
                                pk_column_name = None
                                pk_of_row = None
                                pk_identity = None
                                for item in payload:
                                        if item.startswith('operation='):
                                                operation = item.split('=')[1]
                                        if item.startswith('table='):
                                                table = item.split('=')[1]
                                        if item.startswith('PK name='):
                                                pk_column_name = item.split('=')[1]
                                        if item.startswith('row PK='):
                                                pk_of_row = int(item.split('=')[1])
                                        if item.startswith('person PK='):
                                                try:
                                                        pk_identity = int(item.split('=')[1])
                                                except ValueError:
                                                        _log.exception('error in change notification trigger')
                                                        pk_identity = -1
                                # try sending intra-client signals:
                                # 1) generic signal
                                self.__messages_sent += 1
                                try:
                                        gmDispatcher.send (
                                                signal = notification.channel,
                                                originated_in_database = True,
                                                listener_pid = self.backend_pid,
                                                sending_backend_pid = notification.pid,
                                                pk_identity = pk_identity,
                                                operation = operation,
                                                table = table,
                                                pk_column_name = pk_column_name,
                                                pk_of_row = pk_of_row,
                                                message_index = self.__messages_sent,
                                                notification_index = self.__notifications_received
                                        )
                                except Exception:
                                        print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (notification.channel, notification.pid))
                                        print(sys.exc_info())
                                # 2) dynamically emulated old style table specific signals
                                if table is not None:
                                        self.__messages_sent += 1
                                        signal = '%s_mod_db' % table
                                        _log.debug('emulating old-style table specific signal [%s]', signal)
                                        try:
                                                gmDispatcher.send (
                                                        signal = signal,
                                                        originated_in_database = True,
                                                        listener_pid = self.backend_pid,
                                                        sending_backend_pid = notification.pid,
                                                        pk_identity = pk_identity,
                                                        operation = operation,
                                                        table = table,
                                                        pk_column_name = pk_column_name,
                                                        pk_of_row = pk_of_row,
                                                        message_index = self.__messages_sent,
                                                        notification_index = self.__notifications_received
                                                )
                                        except Exception:
                                                print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (signal, notification.pid))
                                                print(sys.exc_info())

                                # there *may* be more pending notifications but
                                # we don't care when quitting
                                if self._quit_lock.acquire(0):
                                        _have_quit_lock = 1
                                        break

                # exit thread activity
                return

Ancestors

Methods

def shutdown(self)

Cleanly shut down listening.

Unregister notifications. Rejoin listener thread.

Expand source code
def shutdown(self):
        """Cleanly shut down listening.

        Unregister notifications. Rejoin listener thread.
        """
        _log.debug('received %s notifications', self.__notifications_received)
        _log.debug('sent %s messages', self.__messages_sent)
        if self._listener_thread is None:
                self.__shutdown_connection()
                return

        _log.info('stopping backend notifications listener thread')
        self._quit_lock.release()
        try:
                # give the worker thread time to terminate
                self._listener_thread.join(self._poll_interval+2.0)
                try:
                        if self._listener_thread.is_alive():
                                _log.error('listener thread still alive after join()')
                                _log.debug('active threads: %s' % threading.enumerate())
                except Exception:
                        pass
        except Exception:
                print(sys.exc_info())
        self._listener_thread = None
        try:
                self.__unregister_unspecific_notifications()
        except Exception:
                _log.exception('unable to unregister unspecific notifications')

        self.__shutdown_connection()
        return