1+ # MathCAT add-on: generates speech, braille, and allows exploration of expressions written in MathML
2+ # The goal of this add-on is to replicate/improve upon the functionality of MathPlayer which has been discontinued.
3+ # Author: Neil Soiffer
4+ # Copyright: this file is copyright GPL2
5+ # The code additionally makes use of the MathCAT library (written in Rust) which is covered by the MIT license
6+ # and also (obviously) requires external speech engines and braille drivers.
7+ # Note: this code is a lot of cut/paste from other code and very likely could be substantially improved/cleaned.
8+
9+ import braille # we generate braille
10+ import globalPlugins # we are a global plugin
11+ import globalPluginHandler # we are a global plugin
12+ import globalVars
13+ from keyboardHandler import KeyboardInputGesture # navigation key strokes
14+ from logHandler import log # logging
15+ import mathPres # math plugin stuff
16+ from os import path # set rule dir path
17+ import re # regexp patter match
18+ import speech # speech commands
19+ import ui # copy message
20+ from scriptHandler import script # copy MathML via ctrl-c
21+ from synthDriverHandler import getSynth # speech engine param setting
22+ import winUser # clipboard manipultation
23+ from ctypes import windll # register clipboard formats
24+
25+ from . import libmathcat
26+
27+ # speech/SSML processing borrowed from NVDA's mathPres/mathPlayer.py
28+ from speech .commands import (
29+ PitchCommand ,
30+ VolumeCommand ,
31+ RateCommand ,
32+ LangChangeCommand ,
33+ BreakCommand ,
34+ CharacterModeCommand ,
35+ PhonemeCommand ,
36+ )
37+
38+ RE_MP_SPEECH = re .compile (
39+ # Break.
40+ r"<break time='(?P<break>\d+)ms'/> ?"
41+ # Pronunciation of characters.
42+ r"|<say-as interpret-as='characters'>(?P<char>[^<]+)</say-as> ?"
43+ # Specific pronunciation.
44+ r"|<phoneme alphabet='ipa' ph='(?P<ipa>[^']+)'> (?P<phonemeText>[^ <]+)</phoneme> ?"
45+ # Prosody.
46+ r"|<prosody(?: pitch='(?P<pitch>\d+)%')?(?: volume='(?P<volume>\d+)%')?(?: rate='(?P<rate>\d+)%')?> ?"
47+ r"|(?P<prosodyReset></prosody>) ?"
48+ # Other tags, which we don't care about.
49+ r"|<[^>]+> ?"
50+ # Commas indicating pauses in navigation messages.
51+ r"| ?(?P<comma>,) ?"
52+ # Actual content.
53+ r"|(?P<content>[^<,]+)" )
54+
55+ PROSODY_COMMANDS = {
56+ "pitch" : PitchCommand ,
57+ "volume" : VolumeCommand ,
58+ "rate" : RateCommand ,
59+ }
60+
61+ def _processMpSpeech (text , language ):
62+ # MathCAT's default rate is 180 wpm.
63+ # Assume that 0% is 80 wpm and 100% is 450 wpm and scale accordingly.
64+ synth = getSynth ()
65+ wpm = synth ._percentToParam (synth .rate , 80 , 450 )
66+ breakMulti = 180.0 / wpm
67+ out = []
68+ if language :
69+ out .append (LangChangeCommand (language ))
70+ resetProsody = set ()
71+ for m in RE_MP_SPEECH .finditer (text ):
72+ if m .lastgroup == "break" :
73+ out .append (BreakCommand (time = int (m .group ("break" )) * breakMulti ))
74+ elif m .lastgroup == "char" :
75+ out .extend ((CharacterModeCommand (True ), m .group ("char" ), CharacterModeCommand (False )))
76+ elif m .lastgroup == "comma" :
77+ out .append (BreakCommand (time = 100 ))
78+ elif m .lastgroup in PROSODY_COMMANDS :
79+ command = PROSODY_COMMANDS [m .lastgroup ]
80+ out .append (command (multiplier = int (m .group (m .lastgroup )) / 100.0 ))
81+ resetProsody .add (command )
82+ elif m .lastgroup == "prosodyReset" :
83+ for command in resetProsody :
84+ out .append (command (multiplier = 1 ))
85+ resetProsody .clear ()
86+ elif m .lastgroup == "phonemeText" :
87+ out .append (PhonemeCommand (m .group ("ipa" ), text = m .group ("phonemeText" )))
88+ elif m .lastgroup == "content" :
89+ out .append (m .group (0 ))
90+ if language :
91+ out .append (LangChangeCommand (None ))
92+
93+ return out
94+
95+ class MathCATInteraction (mathPres .MathInteractionNVDAObject ):
96+ # Put MathML on the clipboard using the two formats below (defined by MathML spec)
97+ # We use both formats because some apps may only use one or the other
98+ # Note: filed https://github.com/nvaccess/nvda/issues/13240 to make this usable outside of MathCAT
99+ CF_MathML = windll .user32 .RegisterClipboardFormatW ("MathML" )
100+ CF_MathML_Presentation = windll .user32 .RegisterClipboardFormatW ("MathML Presentation" )
101+ # log.info("2**** MathCAT registering data formats: CF_MathML %x, CF_MathML_Presentation %x" % (CF_MathML, CF_MathML_Presentation))
102+
103+ def __init__ (self , provider = None , mathMl = None ):
104+ super (MathCATInteraction , self ).__init__ (provider = provider , mathMl = mathMl )
105+ provider ._setSpeechLanguage (mathMl )
106+ try :
107+ libmathcat .SetMathML (mathMl )
108+ except Exception as e :
109+ speech .speakMessage (_ ("Illegal MathML found: see NVDA error log for details" ))
110+ log .error (e )
111+
112+ def reportFocus (self ):
113+ super (MathCATInteraction , self ).reportFocus ()
114+ try :
115+ speech .speak (_processMpSpeech (libmathcat .GetSpokenText (),
116+ self .provider ._language ))
117+ except Exception as e :
118+ log .error (e )
119+ speech .speakMessage (_ ("Error in speaking math: see NVDA error log for details" ))
120+
121+
122+ def getBrailleRegions (self , review = False ):
123+ yield braille .NVDAObjectRegion (self , appendText = " " )
124+ region = braille .Region ()
125+ region .focusToHardLeft = True
126+ # libmathcat.SetBrailleWidth(braille.handler.displaySize)
127+ try :
128+ region .rawText = libmathcat .GetBraille ("" )
129+ except Exception as e :
130+ log .error (e )
131+
132+ yield region
133+
134+ def getScript (self , gesture ):
135+ # Pass most keys to MathCAT. Pretty ugly.
136+ if isinstance (gesture , KeyboardInputGesture ) and "NVDA" not in gesture .modifierNames and (
137+ gesture .mainKeyName in {
138+ "leftArrow" , "rightArrow" , "upArrow" , "downArrow" ,
139+ "home" , "end" ,
140+ "space" , "backspace" , "enter" ,
141+ }
142+ # or len(gesture.mainKeyName) == 1
143+ ):
144+ return self .script_navigate
145+ return super ().getScript (gesture )
146+
147+ def script_navigate (self , gesture ):
148+ modNames = gesture .modifierNames
149+ try :
150+ text = libmathcat .DoNavigateKeyPress (gesture .vkCode ,
151+ "shift" in modNames , "control" in modNames , "alt" in modNames , False )
152+ speech .speak (_processMpSpeech (text , self .provider ._language ))
153+
154+ # update the braille to reflect the nav position (might be excess code, but it works)
155+ nav_node = libmathcat .GetNavigationMathMLId ()
156+ region = braille .Region ()
157+ region .rawText = libmathcat .GetBraille (nav_node [0 ])
158+ region .focusToHardLeft = True
159+ region .update ()
160+ braille .handler .buffer .regions .append (region )
161+ braille .handler .buffer .focus (region )
162+ braille .handler .buffer .update ()
163+ braille .handler .update ()
164+ except Exception as e :
165+ log .error (e )
166+
167+ _startsWithMath = re .compile ("\s*?<math" )
168+ @script (
169+ gesture = "kb:control+c" ,
170+ )
171+ def script_rawdataToClip (self , gesture ):
172+ try :
173+ mathml = libmathcat .GetNavigationMathML ()[0 ]
174+ if not re .match (self ._startsWithMath , mathml ):
175+ mathml = "<math>" + mathml + "</math>" # copy will fix up namespacing
176+ self ._copyToClipAsMathML (mathml )
177+ ui .message (_ ("copy" ))
178+ except Exception as e :
179+ log .error (e )
180+
181+ # not a perfect match sequence, but should capture normal MathML
182+ _mathTagHasNameSpace = re .compile ("<math .*?xmlns.+?>" )
183+ def _wrapMathMLForClipBoard (self , text : str ) -> str :
184+ # cleanup the MathML a little
185+ mathml_with_ns = text .replace (" data-changed='added'" , "" ).replace (" data-id-added='true'" , "" )
186+ if not re .match (self ._mathTagHasNameSpace , text ):
187+ mathml_with_ns = mathml_with_ns .replace ('math' , 'math xmlns="http://www.w3.org/1998/Math/MathML"' , 1 )
188+ return '<?xml version="1.0"?>' + mathml_with_ns
189+
190+ from typing import Any , Optional
191+ def _copyToClipAsMathML (self , text : str , notify : Optional [bool ] = False ) -> bool :
192+ """Copies the given text to the windows clipboard.
193+ @returns: True if it succeeds, False otherwise.
194+ @param text: the text which will be copied to the clipboard
195+ @param notify: whether to emit a confirmation message
196+ """
197+ # copied from api.py and modified to use CF_MathML_Presentation
198+ if not isinstance (text , str ) or len (text ) == 0 :
199+ return False
200+ import gui
201+ from api import getClipData
202+ try :
203+ with winUser .openClipboard (gui .mainFrame .Handle ):
204+ winUser .emptyClipboard ()
205+ self ._setClipboardData (self .CF_MathML , self ._wrapMathMLForClipBoard (text ))
206+ self ._setClipboardData (self .CF_MathML_Presentation , self ._wrapMathMLForClipBoard (text ))
207+ self ._setClipboardData (winUser .CF_UNICODETEXT , text )
208+ got = getClipData ()
209+ except OSError :
210+ if notify :
211+ ui .reportTextCopiedToClipboard () # No argument reports a failure.
212+ return False
213+ if got == text :
214+ if notify :
215+ ui .reportTextCopiedToClipboard (text )
216+ return True
217+ if notify :
218+ ui .reportTextCopiedToClipboard () # No argument reports a failure.
219+ return False
220+
221+ def _setClipboardData (self , format ,data ):
222+ # Need to support MathML Presentation, so this copied from winUser.py and the first two lines are commented out
223+ # For now only unicode is a supported format
224+ # if format!=CF_UNICODETEXT:
225+ # raise ValueError("Unsupported format")
226+ from textUtils import WCHAR_ENCODING
227+ from ctypes import c_wchar
228+ import winKernel
229+ text = data
230+ bufLen = len (text .encode (WCHAR_ENCODING , errors = "surrogatepass" )) + 2
231+ # Allocate global memory
232+ h = winKernel .HGLOBAL .alloc (winKernel .GMEM_MOVEABLE , bufLen )
233+ # Acquire a lock to the global memory receiving a local memory address
234+ with h .lock () as addr :
235+ # Write the text into the allocated memory
236+ buf = (c_wchar * bufLen ).from_address (addr )
237+ buf .value = text
238+ # Set the clipboard data with the global memory
239+ if not windll .user32 .SetClipboardData (format ,h ):
240+ raise ctypes .WinError ()
241+ # NULL the global memory handle so that it is not freed at the end of scope as the clipboard now has it.
242+ h .forget ()
243+
244+
245+ class MathCAT (mathPres .MathPresentationProvider ):
246+ def __init__ (self ):
247+ # super(MathCAT, self).__init__(*args, **kwargs)
248+
249+ try :
250+ # IMPORTANT -- SetRulesDir must be the first call to libmathcat
251+ rules_dir = path .join ( path .dirname (path .abspath (__file__ )), "Rules" )
252+ log .info ("MathCAT Rules dir: %s" % rules_dir )
253+ libmathcat .SetRulesDir (rules_dir )
254+ libmathcat .SetPreference ("TTS" , "SSML" )
255+
256+ except Exception as e :
257+ log .error (e )
258+
259+ # store mathcontent for navigation and copy
260+ mathcontent = None
261+
262+ def getSpeechForMathMl (self , mathml ):
263+ self ._setSpeechLanguage (mathml )
264+ try :
265+ libmathcat .SetMathML (mathml )
266+ mathcontent = mathml
267+ except Exception as e :
268+ log .error (e )
269+ speech .speakMessage (_ ("Illegal MathML found: see NVDA error log for details" ))
270+ try :
271+ return _processMpSpeech (libmathcat .GetSpokenText (), "en" )
272+ except Exception as e :
273+ log .error (e )
274+ speech .speakMessage (_ ("Error in speaking math: see NVDA error log for details" ))
275+
276+
277+ def getBrailleForMathMl (self , mathml ):
278+ try :
279+ libmathcat .SetMathML (mathml )
280+ except Exception as e :
281+ log .error (e )
282+ speech .speakMessage (_ ("Illegal MathML found: see NVDA error log for details" ))
283+ return libmathcat .GetBraille ("" )
284+ except Exception as e :
285+ log .error (e )
286+ speech .speakMessage (_ ("Error in brailling math: see NVDA error log for details" ))
287+
288+
289+ def interactWithMathMl (self , mathMl ):
290+ MathCATInteraction (provider = self , mathMl = mathMl ).setFocus ()
291+
292+ def _setSpeechLanguage (self , mathMl ):
293+ lang = mathPres .getLanguageFromMath (mathMl )
294+ if not lang :
295+ lang = speech .getCurrentLanguage ()
296+ try :
297+ libmathcat .SetPreference ("Language" , lang .replace ("_" , "-" ))
298+ self ._language = lang
299+ except Exception as e :
300+ log .error (e )
301+
302+
303+ mathPres .registerProvider (MathCAT (), speech = True , braille = True , interaction = True )
304+
305+ class GlobalPlugin (globalPluginHandler .GlobalPlugin ):
306+ def __init__ (self , * args , ** kwargs ):
307+ super ().__init__ (* args , ** kwargs )
308+ MathCAT .__init__ (self )
309+
310+ log .info ("******** message 1 **********" )
0 commit comments