Add "GSAS-query" to main

GSASII/GSASIIctrlGUI.py

@@ -2754,6 +2754,28 @@ def G2MessageBox(parent,msg,title='Error'):
     dlg.Destroy()
 
 ################################################################################
+class G2ModelessMessage(wx.Dialog):
+    '''Simple code to display an infomational message in a non-modal
+    window.
+    '''
+    def __init__(self, parent, message, title):
+        def onClose(event):
+            try:
+                self.Destroy()
+            except:
+                pass
+        super().__init__(parent, title=title, style=wx.DEFAULT_DIALOG_STYLE | wx.STAY_ON_TOP)
+        sizer = wx.BoxSizer(wx.VERTICAL)
+        text = wx.StaticText(self, label=message)
+        text.Wrap(400)
+        sizer.Add(text, 0, wx.ALL | wx.ALIGN_CENTER, 20)
+        close_btn = wx.Button(self, wx.ID_CLOSE, "Close")
+        sizer.Add(close_btn, 0, wx.BOTTOM | wx.ALIGN_CENTER, 15)
+        close_btn.Bind(wx.EVT_BUTTON, onClose)
+        self.Bind(wx.EVT_CLOSE, onClose)
+        self.SetSizerAndFit(sizer)
+        self.Show(True)
+################################################################################
 def findValsInNotebook(data,target):
     'Pull a string of values from saved values in the GSAS-II notebook'
     c = 0
@@ -6245,7 +6267,6 @@ def __init__(self,frame,includeTree=False,morehelpitems=[]):
             helpobj = self.Append(wx.ID_ANY,'Switch to/from branch',
                     'Switch to/from a GSAS-II development branch')
             frame.Bind(wx.EVT_MENU, gitSelectBranch, helpobj)
-        # test if conda present? 
         helpobj = self.Append(wx.ID_ANY,'Add packages for more functionality',
                 'Install optional Python packages to provide more GSAS-II capabilities')
         helpobj.Enable(bool(G2fil.condaRequestList))
@@ -6265,6 +6286,9 @@ def __init__(self,frame,includeTree=False,morehelpitems=[]):
         helpobj = self.Append(wx.ID_ANY,'Help on current data tree item\tF1',
                 'Access web page on selected item in tree')
         frame.Bind(wx.EVT_MENU, self.OnHelpById, id=helpobj.GetId())
+        helpobj = self.Append(wx.ID_ANY,'Help via LLM Docs Search',
+                    'Use LLM to search GSAS-II documentation')
+        frame.Bind(wx.EVT_MENU, LLMsearch, id=helpobj.GetId())
         helpobj = self.Append(wx.ID_ANY,'Citation information',
                 'Show papers that GSAS-II users may wish to cite')
         frame.Bind(wx.EVT_MENU, ShowCitations, id=helpobj.GetId())
@@ -10623,6 +10647,141 @@ def on_char_typed(event):
         dlg.Destroy()
     return val
 
+def InstallLLMindex(G2frame=None):
+    '''Download the documentation index files for LLM with a status dialog.
+    Used with Ollama only
+    '''
+    pdlg = wx.ProgressDialog('Installing Index',
+                            'Downloading and installing index files.\n\n'+
+                            'Search window will open when download is complete',
+                            100,parent=G2frame,style = wx.PD_ELAPSED_TIME)
+    try:
+        pdlg.CenterOnParent()
+        wx.GetApp().Yield()
+        GSASIIpath.getLLMindex()
+    finally:
+        pdlg.Destroy()
+
+def DownloadLLMfiles(G2frame,dlg,installIndex):
+    '''Download the llama model and/or the documentation index files for 
+    LLM searching. This is run in a background thread and only with 
+    the llama backend.
+    '''
+    if installIndex:
+        print('Installing index')
+        GSASIIpath.getLLMindex()
+    print('Installing model')
+    GSASIIpath.installLLamaModel()
+    if dlg:
+        try:
+            wx.CallAfter(dlg.Destroy)
+        except:
+            pass
+    model = GSASIIpath.testLLamaModel()
+    if model is None:
+        print('No model found after install attempt')
+        return
+    wx.CallAfter(LaunchLLama,G2frame)
+
+def LaunchLLama(G2frame):
+    'Start up the Query_gsas LLM dialog with a llama backend'
+    res = GSASIIpath.setupLLama()
+    font = int(14 + GSASII.GSASIIpath.GetConfigValue("FontSize_incr", 0))
+    if res:
+        import gsas_query.gui
+        gsas_query.gui.show_assistant(G2frame,font)
+    else:
+        print('Unable to launch llama')
+
+def LLMsearch(event,repeat=False):
+    '''Master routine to perform LLM searching of documentation. 
+    Checks to see that needed modules and files are present and for 
+    index, recent. Asks user for permission to do installation/downloads 
+    where needed.
+    For llama, where the model is quite large, the download is 
+    performed in a background thread. 
+    '''
+    G2frame = event.GetEventObject().frame
+    while True:
+        res = GSASIIpath.testLLMquery()
+        if res: break
+        dlg = wx.MessageDialog(G2frame,
+                'Packages needed for this are not installed. '+
+                'Do you want to install the packages?',
+                'Install packages',wx.YES_NO | wx.ICON_QUESTION)
+        try:
+            result = dlg.ShowModal()
+        finally:
+            dlg.Destroy()
+        if result == wx.ID_NO: return
+        SelectPkgInstall(event)
+
+    # is the index present or old?
+    age = GSASIIpath.ageLLMindex()
+    installIndex = False
+    if age is None:
+        dlg = wx.MessageDialog(G2frame,
+                'You need the LLM index files. '+
+                'Do you want to download and install them?',
+                'Install index?',wx.YES_NO | wx.ICON_QUESTION)
+        try:
+            result = dlg.ShowModal()
+        finally:
+            dlg.Destroy()
+        if result == wx.ID_NO:
+            return
+        else:
+            installIndex = True
+    elif age > 15:
+        dlg = wx.MessageDialog(G2frame,
+                f'The LLM index files are {age:.1f} days old. '+
+                'You are recommended to update them. '+
+                'Do you want to download and update?',
+                'Install index?',wx.YES_NO | wx.ICON_QUESTION)
+        try:
+            result = dlg.ShowModal()
+        finally:
+            dlg.Destroy()
+        if result != wx.ID_NO:
+            installIndex = True
+    wx.GetApp().Yield()
+    if res == "llama":
+        import threading
+        model = GSASIIpath.testLLamaModel()
+        if model is None:
+            dlg = wx.MessageDialog(G2frame,
+                'You need to install the llama model (once). '+
+                'This is a >2 Gb download that will take a while,\n'+
+                'but GSAS-II can be used while the download is running in the background.\n\n'+
+                'Do you want to download and install this?',
+                'Install model?',wx.YES_NO | wx.ICON_QUESTION)
+            try:
+                result = dlg.ShowModal()
+            finally:
+                dlg.Destroy()
+            if result == wx.ID_NO: return
+            dlg = G2ModelessMessage(G2frame, 
+                'Download(s) are in progress. The "LLM\n'+
+                'Docs Search" window will open when complete.',
+                'Download in progress')
+            thread = threading.Thread(target=DownloadLLMfiles,
+                                      args=(G2frame,dlg,installIndex))
+            thread.start()
+            return
+        else:
+            LaunchLLama(G2frame)
+    elif res == "ollama":
+        font = int(14 + GSASII.GSASIIpath.GetConfigValue("FontSize_incr", 0))
+        if installIndex: InstallLLMindex(G2frame)
+        res = GSASIIpath.setupOllama()
+        if res:
+            import gsas_query.gui
+            gsas_query.gui.show_assistant(G2frame,font)
+        else:
+            print('setupOllama did not complete properly')
+    else:
+        print('Unknown LLM',res)
+
 def HistogramNameTemplate(exporter,stripChars):
     '''Dialog to obtain a string value for grouping histograms
 

GSASII/GSASIIpath.py

@@ -1481,7 +1481,7 @@ def XferConfigIni():
         # Read the configuration file
         cfg.read(cfgfile, encoding='utf-8')
     except Exception as err:
-        print("Error reading {cfgfile}\n",err)
+        print(f"Error reading {cfgfile}\n",err)
         return
 
     # Access values from the configuration file
@@ -1556,6 +1556,8 @@ def findConda():
     We could also look for conda relative to the python (sys.executable)
     image, but I don't want to muck around with python that someone else
     installed.
+
+    Not currently in use.
     '''
     parent = os.path.split(path2GSAS2)[0]
     if sys.platform != "win32":
@@ -2176,6 +2178,189 @@ def openInNewTerm(project=None,g2script=None,pythonapp=sys.executable):
         fp.close()
         subprocess.Popen(cmds,start_new_session=True)
 
+#===========================================================================
+# routines used for Query_gsas (query_gsas2, LLM Documentation searching) 
+def testLLMquery():
+    '''See if the LLM documentation searching is set up to run. 
+    This can take a second or two to run, so don't use this 
+    unless the user asks for use of the LLM query
+
+    :returns: the name of the backend (llama or ollama) if the necessary
+      ingredients are in place, None otherwise
+    '''
+    #import importlib.util # faster but not fast enough
+
+    try:
+        os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+        import chromadb # slowish
+        #if importlib.util.find_spec('chromadb') is None: return
+    except ImportError:
+        #print('chromadb is not installed')
+        from . import GSASIIfiles as G2fil
+        G2fil.NeededPackage({'LLM docs search':['chromadb',
+                                    'llama-cpp-python','huggingface_hub']})
+        return
+    try:
+        import gsas_query.gui
+    except ImportError:
+        print('query_gsas2 is not installed; unexpected!')
+        return
+
+    # is Ollama setup? TODO: should also allow for designation of
+    # Ollama server location and to have 
+    try:
+        if gsas_query.gui._is_ollama_running(): return "ollama"
+    except ImportError:
+        pass
+    bin = os.environ.get("OLLAMA_BIN")
+    if bin is not None:
+        if os.path.exists(bin): return "ollama"
+
+    # No Ollama; is llama installed?
+    try:
+        import llama_cpp # slow
+        try:
+            import huggingface_hub # not strictly necessary
+        except:
+            print('Warning: llama installer but not huggingface_hub.\n'+
+                  'Unable to download llama models')
+        return "llama"
+        #if (importlib.util.find_spec('llama_cpp') is not None and
+        #    importlib.util.find_spec('huggingface_hub') is not None
+        #        ): return "llama"
+    except ImportError:
+        from . import GSASIIfiles as G2fil
+        G2fil.NeededPackage({'LLM docs search':['llama-cpp-python','huggingface_hub']})
+        pass
+    return
+
+def setupOllama():
+    '''Set up to run Ollama
+    '''
+    os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+    import chromadb
+    os.environ['LLM_BACKEND'] ='ollama'
+    return True
+
+def testLLamaModel():
+    '''Test if an llama model has been installed.
+    Returns the name of the most recently installed model 
+    or None if the model needs to be installed.
+    '''
+    modelDir = os.path.expanduser("~/.GSASII/llama_models")
+    if not os.path.exists(modelDir):
+        return None
+    fileList = glob.glob(os.path.join(modelDir,'*.gguf'))
+    if not fileList: return None
+    if len(fileList) == 1:
+        return fileList[0]
+    elif len(fileList) > 1:
+        return sorted(fileList, key=os.path.getmtime, reverse=True)[0]
+    else:
+        return None
+
+def installLLamaModel():
+    '''Download the Qwen2.5-3B-Instruct llama model'''
+    modelDir = os.path.expanduser("~/.GSASII/llama_models")
+    os.makedirs(modelDir, exist_ok=True)
+    print('Downloading a model...')
+    from huggingface_hub import hf_hub_download
+    hf_hub_download(repo_id='Qwen/Qwen2.5-3B-Instruct-GGUF',
+                        filename='qwen2.5-3b-instruct-q4_k_m.gguf',
+                        local_dir=modelDir)
+    print('...Download complete')
+
+def setupLLama():
+    '''Prepare settings to run llama
+    '''
+    os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+    import chromadb
+    model = testLLamaModel()
+#    if model is None:
+#        installLLamaModel()
+#        model = testLLamaModel()
+    if model is None:
+#        print('Error: did not download a llama model from huggingface_hub')
+        print('Error: cannot use llama without a llama model')
+        return False
+    os.environ['LLM_BACKEND'] ='llama_cpp'
+    os.environ['LLAMA_CPP_MODEL'] = model
+    return True
+
+def ageLLMindex():
+    '''Returns the date since the LLM index was last downloaded or
+    None if the file does not exist.
+    '''
+    import datetime
+    f = 'chroma_db'
+    dbdir = os.path.expanduser('~/.GSASII/query_gsas2')
+    db = os.path.join(dbdir,f)
+    if os.path.exists(db):
+        m_time_timestamp = os.path.getmtime(db)
+        last_modified = datetime.datetime.fromtimestamp(m_time_timestamp)
+
+        return (datetime.datetime.now() - last_modified).total_seconds()/(60*60*24)
+
+def getLLMindex():
+    '''Download the ChromaDB database from the GitHub site and place
+    into the location where GSAS-II will use it. 
+
+    Do this in two stages (download to temp location and then move) 
+    as someday perhaps the index update might be performed in the 
+    background.
+
+    TODO: for now this is downlaoded from the GSAS-II site, where the file
+    is placed manually. It should be generated automatically and possibly 
+    on a different site. 
+    '''
+    import tempfile
+    import requests
+    import zipfile
+    import shutil
+
+    OWNER = "AdvancedPhotonSource"
+    #REPO = "Query-GSAS"
+    REPO = "GSAS-II-buildtools"
+    #TAG = "latest-chroma-db"
+    TAG = "v1.0.1"
+    WANTED = "chroma_db_latest.zip"
+    zip_url = f"https://github.com/{OWNER}/{REPO}/releases/download/{TAG}/{WANTED}"
+
+    with requests.Session() as s:
+        # Optional but sometimes helps with GitHub/CDN edge behavior
+        s.headers.update({"User-Agent": "python-requests/zip-downloader"})
+
+        resp = s.get(zip_url, stream=True, timeout=60)
+        resp.raise_for_status()  # fail loudly on 4xx/5xx
+
+        ctype = resp.headers.get("Content-Type", "")
+        if "zip" not in ctype and "octet-stream" not in ctype:
+            # Often means you got HTML (login/error/rate-limit page) instead of the file
+            raise RuntimeError(f"Unexpected content type: {ctype}")
+
+    try:
+        tmp = tempfile.NamedTemporaryFile(suffix=".zip", delete=False)
+        with tmp as f:
+            for chunk in resp.iter_content(chunk_size=1024 * 1024):
+                if chunk:
+                    f.write(chunk)
+        #print("Saved to:", tmp.name)
+        # create the files in a new location and then move them to the final location
+        f = 'chroma_db'
+        finaldir = os.path.expanduser('~/.GSASII/query_gsas2')
+        finaldb = os.path.join(finaldir,f)
+        newdir = os.path.expanduser('~/.GSASII/new_query_gsas2')
+        newdb = os.path.join(newdir,f)
+        os.makedirs(newdir, exist_ok=True)
+        os.makedirs(finaldir, exist_ok=True)
+        with zipfile.ZipFile(tmp.name, 'r') as zip_ref:
+            zip_ref.extractall(newdir)
+        if os.path.exists(finaldb): shutil.rmtree(finaldb)
+        os.rename(newdb,finaldb)
+        if os.path.exists(newdir): shutil.rmtree(newdir)
+    finally:
+        os.unlink(tmp.name)        
+
 if __name__ == '__main__':
     '''What follows is called to update (or downdate) GSAS-II in a
     separate process. This is also called for background tasks such as