diff --git a/GSASII/GSASIIctrlGUI.py b/GSASII/GSASIIctrlGUI.py index 9d253eee..5ba73a13 100644 --- a/GSASII/GSASIIctrlGUI.py +++ b/GSASII/GSASIIctrlGUI.py @@ -2754,6 +2754,28 @@ def G2MessageBox(parent,msg,title='Error'): dlg.Destroy() ################################################################################ +class G2ModelessMessage(wx.Dialog): + '''Simple code to display an infomational message in a non-modal + window. + ''' + def __init__(self, parent, message, title): + def onClose(event): + try: + self.Destroy() + except: + pass + super().__init__(parent, title=title, style=wx.DEFAULT_DIALOG_STYLE | wx.STAY_ON_TOP) + sizer = wx.BoxSizer(wx.VERTICAL) + text = wx.StaticText(self, label=message) + text.Wrap(400) + sizer.Add(text, 0, wx.ALL | wx.ALIGN_CENTER, 20) + close_btn = wx.Button(self, wx.ID_CLOSE, "Close") + sizer.Add(close_btn, 0, wx.BOTTOM | wx.ALIGN_CENTER, 15) + close_btn.Bind(wx.EVT_BUTTON, onClose) + self.Bind(wx.EVT_CLOSE, onClose) + self.SetSizerAndFit(sizer) + self.Show(True) +################################################################################ def findValsInNotebook(data,target): 'Pull a string of values from saved values in the GSAS-II notebook' c = 0 @@ -6245,7 +6267,6 @@ def __init__(self,frame,includeTree=False,morehelpitems=[]): helpobj = self.Append(wx.ID_ANY,'Switch to/from branch', 'Switch to/from a GSAS-II development branch') frame.Bind(wx.EVT_MENU, gitSelectBranch, helpobj) - # test if conda present? helpobj = self.Append(wx.ID_ANY,'Add packages for more functionality', 'Install optional Python packages to provide more GSAS-II capabilities') helpobj.Enable(bool(G2fil.condaRequestList)) @@ -6265,6 +6286,9 @@ def __init__(self,frame,includeTree=False,morehelpitems=[]): helpobj = self.Append(wx.ID_ANY,'Help on current data tree item\tF1', 'Access web page on selected item in tree') frame.Bind(wx.EVT_MENU, self.OnHelpById, id=helpobj.GetId()) + helpobj = self.Append(wx.ID_ANY,'Help via LLM Docs Search', + 'Use LLM to search GSAS-II documentation') + frame.Bind(wx.EVT_MENU, LLMsearch, id=helpobj.GetId()) helpobj = self.Append(wx.ID_ANY,'Citation information', 'Show papers that GSAS-II users may wish to cite') frame.Bind(wx.EVT_MENU, ShowCitations, id=helpobj.GetId()) @@ -10623,6 +10647,141 @@ def on_char_typed(event): dlg.Destroy() return val +def InstallLLMindex(G2frame=None): + '''Download the documentation index files for LLM with a status dialog. + Used with Ollama only + ''' + pdlg = wx.ProgressDialog('Installing Index', + 'Downloading and installing index files.\n\n'+ + 'Search window will open when download is complete', + 100,parent=G2frame,style = wx.PD_ELAPSED_TIME) + try: + pdlg.CenterOnParent() + wx.GetApp().Yield() + GSASIIpath.getLLMindex() + finally: + pdlg.Destroy() + +def DownloadLLMfiles(G2frame,dlg,installIndex): + '''Download the llama model and/or the documentation index files for + LLM searching. This is run in a background thread and only with + the llama backend. + ''' + if installIndex: + print('Installing index') + GSASIIpath.getLLMindex() + print('Installing model') + GSASIIpath.installLLamaModel() + if dlg: + try: + wx.CallAfter(dlg.Destroy) + except: + pass + model = GSASIIpath.testLLamaModel() + if model is None: + print('No model found after install attempt') + return + wx.CallAfter(LaunchLLama,G2frame) + +def LaunchLLama(G2frame): + 'Start up the Query_gsas LLM dialog with a llama backend' + res = GSASIIpath.setupLLama() + font = int(14 + GSASII.GSASIIpath.GetConfigValue("FontSize_incr", 0)) + if res: + import gsas_query.gui + gsas_query.gui.show_assistant(G2frame,font) + else: + print('Unable to launch llama') + +def LLMsearch(event,repeat=False): + '''Master routine to perform LLM searching of documentation. + Checks to see that needed modules and files are present and for + index, recent. Asks user for permission to do installation/downloads + where needed. + For llama, where the model is quite large, the download is + performed in a background thread. + ''' + G2frame = event.GetEventObject().frame + while True: + res = GSASIIpath.testLLMquery() + if res: break + dlg = wx.MessageDialog(G2frame, + 'Packages needed for this are not installed. '+ + 'Do you want to install the packages?', + 'Install packages',wx.YES_NO | wx.ICON_QUESTION) + try: + result = dlg.ShowModal() + finally: + dlg.Destroy() + if result == wx.ID_NO: return + SelectPkgInstall(event) + + # is the index present or old? + age = GSASIIpath.ageLLMindex() + installIndex = False + if age is None: + dlg = wx.MessageDialog(G2frame, + 'You need the LLM index files. '+ + 'Do you want to download and install them?', + 'Install index?',wx.YES_NO | wx.ICON_QUESTION) + try: + result = dlg.ShowModal() + finally: + dlg.Destroy() + if result == wx.ID_NO: + return + else: + installIndex = True + elif age > 15: + dlg = wx.MessageDialog(G2frame, + f'The LLM index files are {age:.1f} days old. '+ + 'You are recommended to update them. '+ + 'Do you want to download and update?', + 'Install index?',wx.YES_NO | wx.ICON_QUESTION) + try: + result = dlg.ShowModal() + finally: + dlg.Destroy() + if result != wx.ID_NO: + installIndex = True + wx.GetApp().Yield() + if res == "llama": + import threading + model = GSASIIpath.testLLamaModel() + if model is None: + dlg = wx.MessageDialog(G2frame, + 'You need to install the llama model (once). '+ + 'This is a >2 Gb download that will take a while,\n'+ + 'but GSAS-II can be used while the download is running in the background.\n\n'+ + 'Do you want to download and install this?', + 'Install model?',wx.YES_NO | wx.ICON_QUESTION) + try: + result = dlg.ShowModal() + finally: + dlg.Destroy() + if result == wx.ID_NO: return + dlg = G2ModelessMessage(G2frame, + 'Download(s) are in progress. The "LLM\n'+ + 'Docs Search" window will open when complete.', + 'Download in progress') + thread = threading.Thread(target=DownloadLLMfiles, + args=(G2frame,dlg,installIndex)) + thread.start() + return + else: + LaunchLLama(G2frame) + elif res == "ollama": + font = int(14 + GSASII.GSASIIpath.GetConfigValue("FontSize_incr", 0)) + if installIndex: InstallLLMindex(G2frame) + res = GSASIIpath.setupOllama() + if res: + import gsas_query.gui + gsas_query.gui.show_assistant(G2frame,font) + else: + print('setupOllama did not complete properly') + else: + print('Unknown LLM',res) + def HistogramNameTemplate(exporter,stripChars): '''Dialog to obtain a string value for grouping histograms diff --git a/GSASII/GSASIIpath.py b/GSASII/GSASIIpath.py index 44e834a4..c15921a3 100644 --- a/GSASII/GSASIIpath.py +++ b/GSASII/GSASIIpath.py @@ -1481,7 +1481,7 @@ def XferConfigIni(): # Read the configuration file cfg.read(cfgfile, encoding='utf-8') except Exception as err: - print("Error reading {cfgfile}\n",err) + print(f"Error reading {cfgfile}\n",err) return # Access values from the configuration file @@ -1556,6 +1556,8 @@ def findConda(): We could also look for conda relative to the python (sys.executable) image, but I don't want to muck around with python that someone else installed. + + Not currently in use. ''' parent = os.path.split(path2GSAS2)[0] if sys.platform != "win32": @@ -2176,6 +2178,189 @@ def openInNewTerm(project=None,g2script=None,pythonapp=sys.executable): fp.close() subprocess.Popen(cmds,start_new_session=True) +#=========================================================================== +# routines used for Query_gsas (query_gsas2, LLM Documentation searching) +def testLLMquery(): + '''See if the LLM documentation searching is set up to run. + This can take a second or two to run, so don't use this + unless the user asks for use of the LLM query + + :returns: the name of the backend (llama or ollama) if the necessary + ingredients are in place, None otherwise + ''' + #import importlib.util # faster but not fast enough + + try: + os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" + import chromadb # slowish + #if importlib.util.find_spec('chromadb') is None: return + except ImportError: + #print('chromadb is not installed') + from . import GSASIIfiles as G2fil + G2fil.NeededPackage({'LLM docs search':['chromadb', + 'llama-cpp-python','huggingface_hub']}) + return + try: + import gsas_query.gui + except ImportError: + print('query_gsas2 is not installed; unexpected!') + return + + # is Ollama setup? TODO: should also allow for designation of + # Ollama server location and to have + try: + if gsas_query.gui._is_ollama_running(): return "ollama" + except ImportError: + pass + bin = os.environ.get("OLLAMA_BIN") + if bin is not None: + if os.path.exists(bin): return "ollama" + + # No Ollama; is llama installed? + try: + import llama_cpp # slow + try: + import huggingface_hub # not strictly necessary + except: + print('Warning: llama installer but not huggingface_hub.\n'+ + 'Unable to download llama models') + return "llama" + #if (importlib.util.find_spec('llama_cpp') is not None and + # importlib.util.find_spec('huggingface_hub') is not None + # ): return "llama" + except ImportError: + from . import GSASIIfiles as G2fil + G2fil.NeededPackage({'LLM docs search':['llama-cpp-python','huggingface_hub']}) + pass + return + +def setupOllama(): + '''Set up to run Ollama + ''' + os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" + import chromadb + os.environ['LLM_BACKEND'] ='ollama' + return True + +def testLLamaModel(): + '''Test if an llama model has been installed. + Returns the name of the most recently installed model + or None if the model needs to be installed. + ''' + modelDir = os.path.expanduser("~/.GSASII/llama_models") + if not os.path.exists(modelDir): + return None + fileList = glob.glob(os.path.join(modelDir,'*.gguf')) + if not fileList: return None + if len(fileList) == 1: + return fileList[0] + elif len(fileList) > 1: + return sorted(fileList, key=os.path.getmtime, reverse=True)[0] + else: + return None + +def installLLamaModel(): + '''Download the Qwen2.5-3B-Instruct llama model''' + modelDir = os.path.expanduser("~/.GSASII/llama_models") + os.makedirs(modelDir, exist_ok=True) + print('Downloading a model...') + from huggingface_hub import hf_hub_download + hf_hub_download(repo_id='Qwen/Qwen2.5-3B-Instruct-GGUF', + filename='qwen2.5-3b-instruct-q4_k_m.gguf', + local_dir=modelDir) + print('...Download complete') + +def setupLLama(): + '''Prepare settings to run llama + ''' + os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" + import chromadb + model = testLLamaModel() +# if model is None: +# installLLamaModel() +# model = testLLamaModel() + if model is None: +# print('Error: did not download a llama model from huggingface_hub') + print('Error: cannot use llama without a llama model') + return False + os.environ['LLM_BACKEND'] ='llama_cpp' + os.environ['LLAMA_CPP_MODEL'] = model + return True + +def ageLLMindex(): + '''Returns the date since the LLM index was last downloaded or + None if the file does not exist. + ''' + import datetime + f = 'chroma_db' + dbdir = os.path.expanduser('~/.GSASII/query_gsas2') + db = os.path.join(dbdir,f) + if os.path.exists(db): + m_time_timestamp = os.path.getmtime(db) + last_modified = datetime.datetime.fromtimestamp(m_time_timestamp) + + return (datetime.datetime.now() - last_modified).total_seconds()/(60*60*24) + +def getLLMindex(): + '''Download the ChromaDB database from the GitHub site and place + into the location where GSAS-II will use it. + + Do this in two stages (download to temp location and then move) + as someday perhaps the index update might be performed in the + background. + + TODO: for now this is downlaoded from the GSAS-II site, where the file + is placed manually. It should be generated automatically and possibly + on a different site. + ''' + import tempfile + import requests + import zipfile + import shutil + + OWNER = "AdvancedPhotonSource" + #REPO = "Query-GSAS" + REPO = "GSAS-II-buildtools" + #TAG = "latest-chroma-db" + TAG = "v1.0.1" + WANTED = "chroma_db_latest.zip" + zip_url = f"https://github.com/{OWNER}/{REPO}/releases/download/{TAG}/{WANTED}" + + with requests.Session() as s: + # Optional but sometimes helps with GitHub/CDN edge behavior + s.headers.update({"User-Agent": "python-requests/zip-downloader"}) + + resp = s.get(zip_url, stream=True, timeout=60) + resp.raise_for_status() # fail loudly on 4xx/5xx + + ctype = resp.headers.get("Content-Type", "") + if "zip" not in ctype and "octet-stream" not in ctype: + # Often means you got HTML (login/error/rate-limit page) instead of the file + raise RuntimeError(f"Unexpected content type: {ctype}") + + try: + tmp = tempfile.NamedTemporaryFile(suffix=".zip", delete=False) + with tmp as f: + for chunk in resp.iter_content(chunk_size=1024 * 1024): + if chunk: + f.write(chunk) + #print("Saved to:", tmp.name) + # create the files in a new location and then move them to the final location + f = 'chroma_db' + finaldir = os.path.expanduser('~/.GSASII/query_gsas2') + finaldb = os.path.join(finaldir,f) + newdir = os.path.expanduser('~/.GSASII/new_query_gsas2') + newdb = os.path.join(newdir,f) + os.makedirs(newdir, exist_ok=True) + os.makedirs(finaldir, exist_ok=True) + with zipfile.ZipFile(tmp.name, 'r') as zip_ref: + zip_ref.extractall(newdir) + if os.path.exists(finaldb): shutil.rmtree(finaldb) + os.rename(newdb,finaldb) + if os.path.exists(newdir): shutil.rmtree(newdir) + finally: + os.unlink(tmp.name) + if __name__ == '__main__': '''What follows is called to update (or downdate) GSAS-II in a separate process. This is also called for background tasks such as diff --git a/GSASII/SUBGROUPS.py b/GSASII/SUBGROUPS.py index 3a3c9f49..3fbaf562 100644 --- a/GSASII/SUBGROUPS.py +++ b/GSASII/SUBGROUPS.py @@ -25,7 +25,10 @@ from . import GSASIIElem as G2elem from . import GSASIIctrlGUI as G2G -import GSASII.Bilbao.BCS_API as BCS +try: + import GSASII.Bilbao.BCS_API as BCS +except ImportError: + BCS = None #bilbaoURL = "http://cryst.ehu.es" #bilbaoSite = f'{bilbaoURL}/cgi-bin/cryst/programs/' @@ -42,6 +45,7 @@ def BCS_init(threadCallback=None): :Returns: True if initialization fails due to the lack of a key ''' + if BCS is None: return True if threadCallback == '': # setup default threading for BCS Post commands def BCS_sleep(): diff --git a/GSASII/gsas_query/__init__.py b/GSASII/gsas_query/__init__.py new file mode 100644 index 00000000..d511f832 --- /dev/null +++ b/GSASII/gsas_query/__init__.py @@ -0,0 +1,20 @@ +""" +GSAS-II Documentation Assistant + +Semantic search + AI answers over GSAS-II tutorials, help pages, and PDFs. +All embedding and retrieval runs locally; the LLM backend is configurable. + +Quick start: + from gsas_query.rag import answer_question + result = answer_question("How do I set up a sequential refinement?", []) + +wxPython GUI (for GSAS-II Help menu integration): + from gsas_query.gui import show_assistant + show_assistant(parent_wx_window) +""" + +from gsas_query._paths import get_chroma_path, get_data_dir, get_static_dir +from gsas_query.rag import answer_question + +__all__ = ["answer_question", "get_chroma_path", "get_data_dir", "get_static_dir"] +__version__ = "0.2.0" diff --git a/GSASII/gsas_query/_paths.py b/GSASII/gsas_query/_paths.py new file mode 100644 index 00000000..ae030e1e --- /dev/null +++ b/GSASII/gsas_query/_paths.py @@ -0,0 +1,25 @@ +"""Central path management for user data (chroma index) and package assets.""" + +import os +from pathlib import Path + +# Suppress HuggingFace tokenizer fork warning and OpenMP threading — +# both must be set before sentence-transformers / loky are imported. +os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") +os.environ.setdefault("OMP_NUM_THREADS", "1") + + +def get_data_dir() -> Path: + """Return (and create) the user data directory for gsas_query.""" + d = Path(os.environ.get("GSAS_QUERY_DATA_DIR", + Path.home() / ".GSASII" / "query_gsas2")) + d.mkdir(parents=True, exist_ok=True) + return d + + +def get_chroma_path() -> str: + return str(get_data_dir() / "chroma_db") + + +def get_static_dir() -> Path: + return Path(__file__).parent / "static" diff --git a/GSASII/gsas_query/_web.py b/GSASII/gsas_query/_web.py new file mode 100644 index 00000000..bb65c09d --- /dev/null +++ b/GSASII/gsas_query/_web.py @@ -0,0 +1,16 @@ +"""Entry point for `gsas-query-web` console script.""" + +import os + + +def main(): + import uvicorn + from .app import app + + host = os.environ.get("HOST", "0.0.0.0") + port = int(os.environ.get("PORT", "8000")) + uvicorn.run(app, host=host, port=port) + + +if __name__ == "__main__": + main() diff --git a/GSASII/gsas_query/app.py b/GSASII/gsas_query/app.py new file mode 100644 index 00000000..a22420ee --- /dev/null +++ b/GSASII/gsas_query/app.py @@ -0,0 +1,154 @@ +""" +FastAPI web server for the GSAS-II documentation chatbot. + + GET / -> chat UI (static/index.html) + GET /health -> {"status": "ok"} + GET /stats -> {"chunks_indexed": N, "llm_backend": "..."} + POST /chat -> RAG query, returns {answer, sources, citations} + POST /ingest -> trigger re-indexing (requires X-Admin-Key header) +""" + +import os +import subprocess +import sys +import time +from typing import Optional + +from fastapi import Depends, FastAPI, HTTPException, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, JSONResponse +from pydantic import BaseModel, Field + +from ._paths import get_static_dir + +STATIC_DIR = get_static_dir() + +app = FastAPI( + title="GSAS-II Documentation Assistant", + description="RAG chatbot over GSAS-II tutorials and help documentation", + version="0.2.0", +) + +origins = os.environ.get("ALLOWED_ORIGINS", "*").split(",") +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_methods=["GET", "POST"], + allow_headers=["*"], +) + + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + return JSONResponse(status_code=500, content={"detail": str(exc)}) + + +# --------------------------------------------------------------------------- # +# Request / response models # +# --------------------------------------------------------------------------- # + +class HistoryTurn(BaseModel): + role: str + content: str + + +class ChatRequest(BaseModel): + message: str = Field(..., max_length=2000) + history: list[HistoryTurn] = Field(default_factory=list, max_length=20) + + +class Source(BaseModel): + title: str + section: str + url: str + category: str + relevance: float + + +class Citation(BaseModel): + title: str + section: str + url: str + relevance: float = 0.0 + + +class ChatResponse(BaseModel): + answer: str + sources: list[Source] + citations: dict[str, Citation] + elapsed_ms: int + + +# --------------------------------------------------------------------------- # +# Simple rate limiting (in-memory, per IP) # +# --------------------------------------------------------------------------- # + +_rate_store: dict[str, list[float]] = {} +RATE_LIMIT = int(os.environ.get("RATE_LIMIT_RPM", "30")) + + +def check_rate_limit(request: Request): + if RATE_LIMIT <= 0: + return + ip = request.client.host if request.client else "unknown" + now = time.time() + timestamps = [t for t in _rate_store.get(ip, []) if now - t < 60] + if len(timestamps) >= RATE_LIMIT: + raise HTTPException(status_code=429, detail="Rate limit exceeded. Please slow down.") + timestamps.append(now) + _rate_store[ip] = timestamps + + +# --------------------------------------------------------------------------- # +# Routes # +# --------------------------------------------------------------------------- # + +@app.get("/", include_in_schema=False) +async def serve_ui(): + return FileResponse(str(STATIC_DIR / "index.html")) + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.get("/stats") +async def stats(): + from .rag import _effective_backend, _get_collection + try: + col = _get_collection() + count = col.count() + except Exception: + count = -1 + return {"chunks_indexed": count, "llm_backend": _effective_backend()} + + +@app.post("/chat", response_model=ChatResponse) +async def chat( + request: ChatRequest, + http_request: Request, + _: None = Depends(check_rate_limit), +): + from .rag import answer_question + + t0 = time.monotonic() + history = [h.model_dump() for h in request.history] + result = answer_question(request.message, history) + elapsed = int((time.monotonic() - t0) * 1000) + + return ChatResponse( + answer=result["answer"], + sources=result["sources"], + citations=result.get("citations", {}), + elapsed_ms=elapsed, + ) + + +@app.post("/ingest") +async def trigger_ingest(http_request: Request): + admin_key = os.environ.get("ADMIN_KEY", "") + if admin_key and http_request.headers.get("X-Admin-Key") != admin_key: + raise HTTPException(status_code=403, detail="Invalid admin key.") + subprocess.Popen([sys.executable, "-m", "gsas_query.ingest"]) + return {"status": "ingestion started in background"} diff --git a/GSASII/gsas_query/cli.py b/GSASII/gsas_query/cli.py new file mode 100644 index 00000000..2650a120 --- /dev/null +++ b/GSASII/gsas_query/cli.py @@ -0,0 +1,215 @@ +""" +GSAS-II Documentation Assistant — command-line interface. + +First-time setup: + gsas-query --setup + gsas-query --setup --reset + gsas-query --setup --html-only + +Ask a single question: + gsas-query "How do I set up a sequential refinement?" + +Interactive REPL: + gsas-query +""" + +import os +import sys +import textwrap + +from ._paths import get_chroma_path + +WIDTH = 80 + + +def _hr(char="─", width=WIDTH): + print(char * width) + + +def _wrap(text: str, indent: int = 0) -> str: + prefix = " " * indent + return textwrap.fill(text, width=WIDTH, initial_indent=prefix, subsequent_indent=prefix) + + +def _print_answer(result: dict): + answer = result.get("answer", "") + sources = result.get("sources", []) + + print() + for para in answer.split("\n"): + if para.strip(): + print(_wrap(para)) + else: + print() + + if sources: + print() + _hr("·") + print("Sources:") + seen = set() + for s in sources: + key = s["url"] + if key in seen: + continue + seen.add(key) + rel = int(s.get("relevance", 0) * 100) + label = f" [{rel}%] {s['title']}" + if s.get("section") and s["section"] != s["title"]: + label += f" › {s['section']}" + print(label) + print(f" {s['url']}") + print() + + +def _collection_count() -> int: + try: + import chromadb + client = chromadb.PersistentClient(path=get_chroma_path()) + return client.get_or_create_collection("gsasii_docs").count() + except Exception: + return 0 + + +def run_setup(reset: bool = False, html_only: bool = False): + from . import ingest + import argparse + + print("Starting ingestion. This may take 10–20 minutes.") + print(f"Index will be stored at: {get_chroma_path()}\n") + + argv_backup = sys.argv + sys.argv = ["gsas-query"] + if reset: + sys.argv.append("--reset") + if html_only: + sys.argv.append("--html-only") + try: + ingest.main() + finally: + sys.argv = argv_backup + + +def ask(question: str, history: list | None = None) -> dict: + from .rag import answer_question + return answer_question(question, history or []) + + +def interactive(): + from .rag import _effective_backend + count = _collection_count() + if count == 0: + print("\nWarning: the knowledge base is empty.") + print("Run: gsas-query --setup\n") + else: + print(f"\nKnowledge base: {count:,} indexed chunks.") + + backend = _effective_backend() + print(f"LLM backend: {backend}") + print("Type your question and press Enter. 'clear' resets history, 'quit' exits.\n") + _hr() + + history: list[dict] = [] + + while True: + try: + question = input("You: ").strip() + except (EOFError, KeyboardInterrupt): + print("\nGoodbye.") + break + + if not question: + continue + if question.lower() in {"quit", "exit", "q"}: + print("Goodbye.") + break + if question.lower() in {"clear", "reset"}: + history.clear() + print("(History cleared)\n") + continue + + print("Thinking…", end="", flush=True) + try: + result = ask(question, history) + except Exception as e: + print(f"\rError: {e}\n") + continue + + print("\r" + " " * 12 + "\r", end="") + print("Assistant:", end="") + _print_answer(result) + + history.append({"role": "user", "content": question}) + history.append({"role": "assistant", "content": result.get("answer", "")}) + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description="GSAS-II Documentation Assistant", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("question", nargs="?", help="Question to ask (omit for interactive mode)") + parser.add_argument("--setup", action="store_true", help="Index documentation (first-time setup)") + parser.add_argument("--reset", action="store_true", help="Drop and rebuild the index") + parser.add_argument("--html-only", action="store_true", help="Skip PDFs during setup") + parser.add_argument("--backend", choices=["ollama", "anthropic", "retrieval", "llama_cpp"], + help="Override LLM_BACKEND env var") + parser.add_argument("--model", help="Override OLLAMA_MODEL or ANTHROPIC_MODEL") + parser.add_argument("--stats", action="store_true", help="Show index statistics and exit") + parser.add_argument("--gui", action="store_true", help="Open the wxPython desktop assistant") + args = parser.parse_args() + + if args.backend: + os.environ["LLM_BACKEND"] = args.backend + if args.model: + effective = os.environ.get("LLM_BACKEND", "") + if effective == "anthropic": + os.environ["ANTHROPIC_MODEL"] = args.model + elif effective == "llama_cpp": + os.environ["LLAMA_CPP_MODEL"] = args.model + else: + os.environ["OLLAMA_MODEL"] = args.model + + print("GSAS-II Documentation Assistant") + _hr("═") + + if args.stats: + from .rag import _effective_backend + count = _collection_count() + print(f"Indexed chunks : {count:,}") + print(f"LLM backend : {_effective_backend()}") + print(f"Chroma DB : {get_chroma_path()}") + return + + if args.setup: + run_setup(reset=args.reset, html_only=args.html_only) + return + + if args.gui: + from .gui import show_assistant + try: + import wx + except ImportError: + print("wxPython is required for the GUI. Install it or use GSAS-II's Python.") + sys.exit(1) + app = wx.App(False) + show_assistant() + app.MainLoop() + return + + if args.question: + count = _collection_count() + if count == 0: + print("Knowledge base is empty. Run: gsas-query --setup") + sys.exit(1) + print(f"({count:,} chunks indexed)\n") + result = ask(args.question) + _print_answer(result) + else: + interactive() + + +if __name__ == "__main__": + main() diff --git a/GSASII/gsas_query/gui.py b/GSASII/gsas_query/gui.py new file mode 100644 index 00000000..8eb54e92 --- /dev/null +++ b/GSASII/gsas_query/gui.py @@ -0,0 +1,690 @@ +""" +GSAS-II Documentation Assistant — wxPython desktop dialog. + +Standalone: + gsas-query --gui + python -m gsas_query.gui + +GSAS-II Help menu integration: + def OnDocAssistant(self, event): + from gsas_query.gui import show_assistant + show_assistant(self) + +Uses MathJaX to display equations using https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js, if available. +""" + +import atexit +import html +import os +import re +import subprocess +import sys +import threading +import time +import webbrowser + +from ._paths import get_chroma_path + + +def _get_gsas_font_size(default: int = 10) -> int: + """Return the font size to use, in points. + + Priority: + 1. specified font size in show_assistant() or GSASQueryDialog() call + 2. GSAS_QUERY_FONT_SIZE env var (explicit override) + 3. *default* (10 pt) + """ + env = os.environ.get("GSAS_QUERY_FONT_SIZE") + if env: + try: + return int(env) + except ValueError: + pass + return default + + +def _ollama_url() -> str: + return os.environ.get("OLLAMA_URL", "http://localhost:11434") + + +def _format_llm_markdown(text: str) -> str: + """Convert markdown/math flavored model output into readable plain text. + + The chat transcript is rendered in a wx.TextCtrl, so we normalize common + markdown and MathJax delimiters rather than showing raw markup. + """ + if not text: + return "" + + out = text.replace("\r\n", "\n") + + # Avoid duplicated speaker prefix in the transcript. + if out.lstrip().lower().startswith("assistant:"): + out = out.split(":", 1)[1].lstrip() + + # Convert fenced code blocks into plain indented blocks. + out = re.sub(r"```(?:[\w+-]+)?\n(.*?)```", lambda m: "\n" + "\n".join( + f" {line}" if line else "" for line in m.group(1).split("\n") + ) + "\n", out, flags=re.DOTALL) + + # Convert display math delimiters into visible equation lines. + out = out.replace("\\[", "\n") + out = out.replace("\\]", "\n") + out = out.replace("$$", "\n") + + # Convert inline math delimiters to simple quoted equation text. + out = out.replace("\\(", "[") + out = out.replace("\\)", "]") + + # Simplify common markdown markers for plain-text display. + out = re.sub(r"^\s*[-*]\s+", "• ", out, flags=re.MULTILINE) + out = re.sub(r"\*\*(.*?)\*\*", r"\1", out) + out = re.sub(r"__(.*?)__", r"\1", out) + out = re.sub(r"`([^`]+)`", r"\1", out) + + # Collapse excess blank space introduced by conversions. + out = re.sub(r"\n{3,}", "\n\n", out) + return out.strip() + + +def _strip_speaker_prefix(text: str) -> str: + """Drop a leading 'Assistant:' emitted by some models.""" + if text.lstrip().lower().startswith("assistant:"): + return text.split(":", 1)[1].lstrip() + return text + + +def _inline_md_to_html(line: str) -> str: + """Render a small markdown subset into HTML-safe inline content.""" + out = html.escape(line) + out = re.sub(r"`([^`]+)`", r"\1", out) + out = re.sub(r"\*\*(.+?)\*\*", r"\1", out) + out = re.sub(r"__(.+?)__", r"\1", out) + return out + + +def _assistant_text_to_html(text: str) -> str: + """Convert model output into safe HTML while preserving TeX delimiters.""" + if not text: + return "" + + src = _strip_speaker_prefix(text).replace("\r\n", "\n") + parts = re.split(r"```(?:[\w+-]+)?\n(.*?)```", src, flags=re.DOTALL) + chunks: list[str] = [] + + for i, part in enumerate(parts): + if i % 2 == 1: + chunks.append(f"
{html.escape(part.strip())}
") + continue + + lines = part.split("\n") + in_list = False + para_lines: list[str] = [] + + def flush_para(): + if para_lines: + body = "
".join(_inline_md_to_html(x) for x in para_lines) + chunks.append(f"

{body}

") + para_lines.clear() + + for raw in lines: + line = raw.rstrip() + m = re.match(r"^\s*[-*]\s+(.*)$", line) + if m: + flush_para() + if not in_list: + chunks.append("") + in_list = False + continue + + if in_list: + chunks.append("") + in_list = False + para_lines.append(line) + + flush_para() + if in_list: + chunks.append("") + + return "\n".join(chunks) + + +def _chat_document(messages: list[dict], font_size: int) -> str: + """Build the full chat HTML document with MathJax enabled.""" + msg_html: list[str] = [] + for msg in messages: + role = msg.get("role", "assistant") + if role == "user": + body = f"

{_inline_md_to_html(msg.get('content', ''))}

" + msg_html.append( + f'
You
{body}
' + ) + elif role == "system": + body = f"

{_inline_md_to_html(msg.get('content', ''))}

" + msg_html.append( + f'
{body}
' + ) + else: + body = _assistant_text_to_html(msg.get("content", "")) + msg_html.append( + f'
Assistant
{body}
' + ) + + if not msg_html: + msg_html.append('

Ask a question about GSAS-II documentation.

') + + return f""" + + + + + + + + +
{''.join(msg_html)}
+ + + +""" + + +def _ollama_bin() -> str: + """Return ollama executable path (optionally overridden by env).""" + return os.environ.get("OLLAMA_BIN", "ollama") + + +def _is_ollama_running() -> bool: + try: + import httpx + return httpx.get(f"{_ollama_url()}/api/tags", timeout=2).status_code == 200 + except Exception: + return False + + +def _launch_ollama() -> "subprocess.Popen | None": + """Start `ollama serve` if not already running. Returns the process we started, or None.""" + if _is_ollama_running(): + return None + repeat = True + cmdarg = "serve" + sec = 6 + while repeat: + repeat = False + try: + proc = subprocess.Popen( + [_ollama_bin(), cmdarg], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + encoding='UTF-8' + ) + except FileNotFoundError: + return None + for _ in range(sec*2): # wait up to `sec` s + time.sleep(0.5) + if _is_ollama_running(): + # Register a fallback: if GSAS-II exits without closing the dialog + # (EVT_CLOSE won't fire for child frames), atexit still kills Ollama. + atexit.register(proc.terminate) + return proc + elif cmdarg == "serve": + # on Mac provided binary runs w/o server. Longer timeout, as + # one may need to respond to GUI questions + for line in proc.stderr: + if "serve command not supported" in line: + repeat = True + sec = 60 + break + else: + continue + break + print(f'Failed to launch Ollama server with "{_ollama_bin()} {cmdarg}"') + cmdarg = '' # if repeat, try again without serve + proc.terminate() + return None + +try: + import wx + import wx.html + try: + import wx.html2 + _HAS_WEBVIEW = True + except Exception: + _HAS_WEBVIEW = False +except ImportError: + print("wxPython is required for the GUI. It is included with GSAS-II.") + sys.exit(1) + + +# ── Colour constants ─────────────────────────────────────────────────────────── + +_BLUE = wx.Colour(26, 79, 138) # Argonne blue +_ACCENT = wx.Colour(232, 119, 34) # APS orange +_BG = wx.Colour(244, 246, 249) +_BOT_BG = wx.Colour(255, 255, 255) +_BORDER = wx.Colour(209, 217, 230) +_MUTED = wx.Colour(107, 114, 128) + + +# ── Background query thread ──────────────────────────────────────────────────── + +class _QueryThread(threading.Thread): + def __init__(self, parent, question: str, history: list): + super().__init__(daemon=True) + self.parent = parent + self.question = question + self.history = list(history) + + def run(self): + try: + from .rag import answer_question + result = answer_question(self.question, self.history) + except Exception as e: + result = {"answer": f"Error: {e}", "sources": []} + wx.CallAfter(self.parent._on_query_done, result) + + +# ── Source link panel ────────────────────────────────────────────────────────── + +class _SourcePanel(wx.Panel): + def __init__(self, parent, source: dict, number: int, font_size: int = 10): + super().__init__(parent, style=wx.BORDER_NONE) + self.SetBackgroundColour(parent.GetBackgroundColour()) + + url = source.get("url", "") + title = source.get("title", "Unknown") + section = source.get("section", "") + rel = int(source.get("relevance", 0) * 100) + + small = max(font_size - 1, 8) + + # Number label — bold, matches inline [N] in answer text + num_lbl = wx.StaticText(self, label=f"[{number}]") + num_lbl.SetForegroundColour(_ACCENT) + num_lbl.SetFont(wx.Font(wx.FontInfo(small).Bold())) + + # Clickable title + section + text = title + if section and section != title: + text += f" › {section}" + text += f" [{rel}%]" + + lnk = wx.StaticText(self, label=text) + lnk.SetForegroundColour(_BLUE) + lnk.SetCursor(wx.Cursor(wx.CURSOR_HAND)) + lnk.SetFont(wx.Font(wx.FontInfo(small))) + lnk.Bind(wx.EVT_LEFT_UP, lambda e: webbrowser.open(url)) + + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(num_lbl, 0, wx.ALIGN_CENTER_VERTICAL | wx.RIGHT, 6) + sizer.Add(lnk, 1, wx.ALIGN_CENTER_VERTICAL) + self.SetSizer(sizer) + + +# ── Main dialog ──────────────────────────────────────────────────────────────── + +class GSASQueryDialog(wx.Frame): + """ + Modeless frame — stays open while the user works in GSAS-II. + Call show_assistant() rather than instantiating directly. + """ + + def __init__(self, parent, fontsize=None): + super().__init__( + parent, + title="GSAS-II Documentation Assistant", + size=(700, 620), + style=wx.DEFAULT_FRAME_STYLE | wx.FRAME_FLOAT_ON_PARENT, + ) + self._history: list[dict] = [] + self._messages: list[dict] = [] + self._pending_question = "" + self._busy = False + self._ollama_proc: "subprocess.Popen | None" = None + if fontsize is None: + self._font_size = _get_gsas_font_size(default=10) + else: + self._font_size = fontsize + self._build_ui() + if self._chat_web is None: + self._append_system( + "Math rendering unavailable: WebView backend is not available in this wxPython environment." + ) + self.Centre() + self.Bind(wx.EVT_CLOSE, self._on_close) + self._check_index() + self._ensure_ollama() + + # ── UI construction ──────────────────────────────────────────────────────── + + def _build_ui(self): + self.SetBackgroundColour(_BG) + outer = wx.BoxSizer(wx.VERTICAL) + + # Header + header = wx.Panel(self) + header.SetBackgroundColour(_BLUE) + h_sizer = wx.BoxSizer(wx.HORIZONTAL) + title_lbl = wx.StaticText(header, label="GSAS-II Documentation Assistant") + title_lbl.SetForegroundColour(wx.WHITE) + f = title_lbl.GetFont() + f.SetWeight(wx.FONTWEIGHT_BOLD) + f.SetPointSize(f.GetPointSize() + 2) + title_lbl.SetFont(f) + self._index_lbl = wx.StaticText(header, label="") + self._index_lbl.SetForegroundColour(wx.Colour(200, 220, 255)) + small = self._index_lbl.GetFont() + small.SetPointSize(small.GetPointSize() - 1) + self._index_lbl.SetFont(small) + h_sizer.Add(title_lbl, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT, 14) + h_sizer.AddStretchSpacer() + h_sizer.Add(self._index_lbl, 0, wx.ALIGN_CENTER_VERTICAL | wx.RIGHT, 14) + header.SetSizer(h_sizer) + header.SetMinSize((-1, 46)) + outer.Add(header, 0, wx.EXPAND) + + # Chat transcript (WebView + MathJax when available) + self._chat_web = None + self._chat = None + if _HAS_WEBVIEW: + try: + self._chat_web = wx.html2.WebView.New(self) + outer.Add(self._chat_web, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 10) + self._render_chat() + except Exception: + # Some wx builds expose wx.html2 but lack a working runtime backend. + self._chat_web = None + + if self._chat_web is None: + self._chat = wx.TextCtrl( + self, style=wx.TE_MULTILINE | wx.TE_READONLY | wx.TE_RICH2 | wx.BORDER_NONE + ) + self._chat.SetBackgroundColour(_BOT_BG) + self._chat.SetMinSize((-1, 300)) + outer.Add(self._chat, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 10) + + # Sources panel + src_box = wx.StaticBox(self, label="Sources") + src_box.SetForegroundColour(_MUTED) + self._src_sizer = wx.StaticBoxSizer(src_box, wx.VERTICAL) + self._src_panel = wx.ScrolledWindow(self, style=wx.BORDER_NONE) + self._src_panel.SetScrollRate(0, 12) + self._src_panel.SetBackgroundColour(_BG) + self._src_inner = wx.BoxSizer(wx.VERTICAL) + self._src_panel.SetSizer(self._src_inner) + self._src_sizer.Add(self._src_panel, 1, wx.EXPAND | wx.ALL, 4) + self._src_panel.SetMinSize((-1, 90)) + outer.Add(self._src_sizer, 0, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 10) + + # Progress indicator — shown while the LLM is generating + self._gauge = wx.Gauge(self, range=50, style=wx.GA_HORIZONTAL | wx.GA_SMOOTH) + self._gauge.Hide() + outer.Add(self._gauge, 0, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 10) + self._gauge_timer = wx.Timer(self) + self.Bind(wx.EVT_TIMER, lambda _: self._gauge.Pulse(), self._gauge_timer) + + # Input row + in_sizer = wx.BoxSizer(wx.HORIZONTAL) + self._input = wx.TextCtrl( + self, style=wx.TE_PROCESS_ENTER | wx.TE_MULTILINE, size=(-1, 54), + ) + self._input.SetHint("Ask a question about GSAS-II…") + self._input.Bind(wx.EVT_TEXT_ENTER, self._on_send) + self._input.Bind(wx.EVT_KEY_DOWN, self._on_key) + + self._send_btn = wx.Button(self, label="Ask", size=(70, 54)) + self._send_btn.SetBackgroundColour(_BLUE) + self._send_btn.SetForegroundColour(wx.WHITE) + self._send_btn.Bind(wx.EVT_BUTTON, self._on_send) + + clear_btn = wx.Button(self, label="Clear", size=(60, 54)) + clear_btn.Bind(wx.EVT_BUTTON, self._on_clear) + + in_sizer.Add(self._input, 1, wx.EXPAND | wx.RIGHT, 6) + in_sizer.Add(self._send_btn, 0) + in_sizer.Add(clear_btn, 0, wx.LEFT, 4) + outer.Add(in_sizer, 0, wx.EXPAND | wx.ALL, 10) + + self._status = self.CreateStatusBar() + self.SetSizer(outer) + self.Layout() + + # ── Index check ──────────────────────────────────────────────────────────── + + def _check_index(self): + def _check(): + try: + import chromadb + client = chromadb.PersistentClient(path=get_chroma_path()) + count = client.get_or_create_collection("gsasii_docs").count() + except Exception: + count = 0 + wx.CallAfter(self._set_index_status, count) + threading.Thread(target=_check, daemon=True).start() + + def _ensure_ollama(self): + from .rag import _effective_backend + if _effective_backend() != "ollama": + return + + # If an Ollama server is already reachable (possibly started outside this env), + # do not launch another process from PATH. + if _is_ollama_running(): + self._status.SetStatusText("Connected to Ollama") + return + + def _start(): + wx.CallAfter(self._status.SetStatusText, "Checking Ollama…") + proc = _launch_ollama() + if proc is not None: + self._ollama_proc = proc + wx.CallAfter(self._status.SetStatusText, "Ollama started") + else: + wx.CallAfter( + self._status.SetStatusText, + "Ollama not found. Start it manually: ollama serve", + ) + + + threading.Thread(target=_start, daemon=True).start() + + def _on_close(self, event): + global _instance + _instance = None + if self._ollama_proc is not None: + atexit.unregister(self._ollama_proc.terminate) # cancel the fallback + self._ollama_proc.terminate() + self._ollama_proc = None + self.Destroy() + + def _set_index_status(self, count: int): + if count == 0: + self._index_lbl.SetLabel("Not indexed — run: gsas-query --setup") + self._append_system( + "The knowledge base is empty.\n" + "Run 'gsas-query --setup' to index all GSAS-II documentation (~10 min)." + ) + else: + from .rag import _effective_backend + self._index_lbl.SetLabel(f"{count:,} chunks · {_effective_backend()}") + + # ── Event handlers ───────────────────────────────────────────────────────── + + def _on_key(self, event): + if event.GetKeyCode() == wx.WXK_RETURN and not event.ShiftDown(): + self._on_send(event) + else: + event.Skip() + + def _on_send(self, event): + question = self._input.GetValue().strip() + if not question or self._busy: + return + self._input.SetValue("") + self._pending_question = question + self._busy = True + self._send_btn.Disable() + self._status.SetStatusText("Thinking…") + self._gauge.Show() + self._gauge_timer.Start(80) # pulse every 80 ms + wx.BeginBusyCursor() + self.Layout() + self._append_user(question) + self._clear_sources() + _QueryThread(self, question, self._history).start() + + def _on_clear(self, event): + self._history.clear() + self._messages.clear() + if self._chat_web is not None: + self._render_chat() + else: + self._chat.SetValue("") + self._clear_sources() + self._status.SetStatusText("History cleared.") + + def _on_query_done(self, result: dict): + self._busy = False + self._send_btn.Enable() + self._gauge_timer.Stop() + self._gauge.Hide() + wx.EndBusyCursor() + self.Layout() + self._status.SetStatusText("") + answer = result.get("answer", "") + self._append_assistant(answer) + self._history.append({"role": "user", "content": self._pending_question}) + self._history.append({"role": "assistant", "content": answer}) + self._show_sources(result.get("citations", {})) + + # ── Chat helpers ─────────────────────────────────────────────────────────── + + def _render_chat(self): + if self._chat_web is not None: + self._chat_web.SetPage(_chat_document(self._messages, self._font_size), "") + + def _append_user(self, text: str): + self._messages.append({"role": "user", "content": text}) + if self._chat_web is not None: + self._render_chat() + return + self._chat.SetDefaultStyle( + wx.TextAttr(_BLUE, font=wx.Font(wx.FontInfo(self._font_size).Bold())) + ) + self._chat.AppendText(f"You: {text}\n\n") + self._chat.SetDefaultStyle(wx.TextAttr(wx.BLACK)) + + def _append_assistant(self, text: str): + self._messages.append({"role": "assistant", "content": text}) + if self._chat_web is not None: + self._render_chat() + return + formatted = _format_llm_markdown(text) + self._chat.SetDefaultStyle( + wx.TextAttr(wx.BLACK, font=wx.Font(wx.FontInfo(self._font_size))) + ) + self._chat.AppendText(f"Assistant: {formatted}\n\n") + self._chat.SetDefaultStyle(wx.TextAttr(wx.BLACK)) + + def _append_system(self, text: str): + self._messages.append({"role": "system", "content": text}) + if self._chat_web is not None: + self._render_chat() + return + self._chat.SetDefaultStyle(wx.TextAttr(_MUTED)) + self._chat.AppendText(f"{text}\n\n") + self._chat.SetDefaultStyle(wx.TextAttr(wx.BLACK)) + + # ── Source helpers ───────────────────────────────────────────────────────── + + def _clear_sources(self): + self._src_inner.Clear(delete_windows=True) + self._src_panel.Layout() + + def _show_sources(self, citations: dict): + self._clear_sources() + for key in sorted(citations, key=lambda k: int(k)): + item = _SourcePanel(self._src_panel, citations[key], + number=int(key), font_size=self._font_size) + self._src_inner.Add(item, 0, wx.EXPAND | wx.BOTTOM, 4) + self._src_panel.FitInside() + self._src_panel.Layout() + self.Layout() + + +# ── Public API ───────────────────────────────────────────────────────────────── + +_instance: GSASQueryDialog | None = None + + +def show_assistant(parent=None,fontsize=None) -> GSASQueryDialog: + """ + Show the GSAS-II Documentation Assistant dialog. + + Call from GSAS-II's Help menu: + from gsas_query.gui import show_assistant + show_assistant(self) + + If already open, brings the window to front instead of opening a duplicate. + """ + global _instance + if _instance is None or not _instance.IsShown(): + _instance = GSASQueryDialog(parent,fontsize=fontsize) + _instance.Show() + else: + _instance.Raise() + return _instance + + +# ── Standalone entry point ───────────────────────────────────────────────────── + +if __name__ == "__main__": + app = wx.App(False) + show_assistant() + app.MainLoop() diff --git a/GSASII/gsas_query/ingest.py b/GSASII/gsas_query/ingest.py new file mode 100644 index 00000000..d79fbcca --- /dev/null +++ b/GSASII/gsas_query/ingest.py @@ -0,0 +1,267 @@ +""" +Ingestion pipeline: fetch GSAS-II tutorials (HTML + PDFs), +chunk by section, embed with sentence-transformers, store in ChromaDB. +""" + +import argparse +import hashlib +import io +import multiprocessing +import re +import sys +import time + +from ._paths import get_chroma_path + +COLLECTION_NAME = "gsasii_docs" +MAX_CHUNK_CHARS = 1200 +OVERLAP_CHARS = 150 +REQUEST_DELAY = 0.5 + + +def get_collection(reset: bool = False): + import chromadb + client = chromadb.PersistentClient(path=get_chroma_path()) + if reset: + try: + client.delete_collection(COLLECTION_NAME) + print("Dropped existing collection.") + except Exception: + pass + return client.get_or_create_collection( + name=COLLECTION_NAME, + metadata={"hnsw:space": "cosine"}, + ) + + +def chunk_text(text: str, max_chars: int = MAX_CHUNK_CHARS, overlap: int = OVERLAP_CHARS) -> list[str]: + text = re.sub(r"\s+", " ", text).strip() + if len(text) <= max_chars: + return [text] if text else [] + + chunks = [] + start = 0 + while start < len(text): + end = min(start + max_chars, len(text)) + if end < len(text): + boundary = max( + text.rfind(". ", start, end), + text.rfind(".\n", start, end), + text.rfind("! ", start, end), + text.rfind("? ", start, end), + ) + if boundary > start + max_chars // 2: + end = boundary + 1 + chunk = text[start:end].strip() + if chunk: + chunks.append(chunk) + if end >= len(text): + break + start = end - overlap + return chunks + + +def extract_html_sections(html: str, source_title: str) -> list[dict]: + from bs4 import BeautifulSoup, Tag + soup = BeautifulSoup(html, "html.parser") + + for tag in soup(["script", "style", "nav", "footer", "img"]): + tag.decompose() + + body = soup.find("body") or soup + sections = [] + current_heading = source_title + current_text_parts: list[str] = [] + heading_tags = {"h1", "h2", "h3", "h4"} + + def flush(): + text = " ".join(current_text_parts).strip() + if text and len(text) > 80: + sections.append({"heading": current_heading, "text": text}) + + for elem in body.descendants: + if not isinstance(elem, Tag): + continue + if elem.name in heading_tags: + flush() + current_heading = elem.get_text(separator=" ", strip=True) + current_text_parts = [] + elif elem.name in {"p", "li", "td", "th", "pre", "blockquote", "dd", "dt"}: + txt = elem.get_text(separator=" ", strip=True) + if txt: + current_text_parts.append(txt) + + flush() + return sections + + +def ingest_html_source(source: dict, collection, model): + import requests + url = source["url"] + title = source["title"] + category = source["category"] + + print(f" Fetching: {title} ({url})") + try: + resp = requests.get(url, timeout=30) + resp.raise_for_status() + except Exception as e: + print(f" ERROR fetching {url}: {e}") + return 0 + + sections = extract_html_sections(resp.text, title) + # Use a dict keyed by doc_id to deduplicate identical chunks within the source. + records: dict[str, tuple] = {} + + for section in sections: + chunks = chunk_text(section["text"]) + for chunk in chunks: + doc_id = hashlib.md5(f"{url}|{section['heading']}|{chunk}".encode()).hexdigest() + if doc_id not in records: + records[doc_id] = (chunk, model([chunk])[0], { + "url": url, + "title": title, + "section": section["heading"], + "category": category, + "source_type": "html", + }) + + if records: + ids = list(records) + docs = [v[0] for v in records.values()] + embeddings = [v[1] for v in records.values()] + metadatas = [v[2] for v in records.values()] + collection.upsert(ids=ids, documents=docs, embeddings=embeddings, metadatas=metadatas) + print(f" -> {len(ids)} chunks stored") + time.sleep(REQUEST_DELAY) + return len(ids) + + time.sleep(REQUEST_DELAY) + return 0 + + +def ingest_pdf_source(source: dict, collection, model): + try: + from pypdf import PdfReader + except ImportError: + print(" pypdf not installed, skipping PDF ingestion.") + return 0 + + import requests + url = source["url"] + title = source["title"] + category = source["category"] + + print(f" Fetching PDF: {title}") + try: + resp = requests.get(url, timeout=60) + resp.raise_for_status() + except Exception as e: + print(f" ERROR fetching PDF {url}: {e}") + return 0 + + try: + reader = PdfReader(io.BytesIO(resp.content)) + except Exception as e: + print(f" ERROR parsing PDF: {e}") + return 0 + + ids, docs, embeddings, metadatas = [], [], [], [] + total_pages = len(reader.pages) + print(f" {total_pages} pages") + + for page_start in range(0, total_pages, 3): + page_end = min(page_start + 3, total_pages) + combined_text = "" + for p in range(page_start, page_end): + combined_text += (reader.pages[p].extract_text() or "") + "\n" + + section_label = f"Pages {page_start + 1}-{page_end}" + for i, chunk in enumerate(chunk_text(combined_text)): + doc_id = hashlib.md5(f"{url}|{section_label}|{i}".encode()).hexdigest() + embedding = model([chunk])[0] + ids.append(doc_id) + docs.append(chunk) + embeddings.append(embedding) + metadatas.append({ + "url": url, + "title": title, + "section": section_label, + "category": category, + "source_type": "pdf", + }) + + if ids: + collection.upsert(ids=ids, documents=docs, embeddings=embeddings, metadatas=metadatas) + print(f" -> {len(ids)} chunks stored") + + return len(ids) + + +def main(): + parser = argparse.ArgumentParser(description="Ingest GSAS-II docs into ChromaDB") + parser.add_argument("--html-only", action="store_true", help="Skip PDF ingestion") + parser.add_argument("--reset", action="store_true", help="Drop and rebuild collection") + parser.add_argument("--book", action="store_true", + help="Include Powder Diffraction Crystallography book (185 HTML pages)") + parser.add_argument("--manual", action="store_true", + help="Include Programmers' Manual (24 HTML chapters)") + args = parser.parse_args() + + from chromadb.utils.embedding_functions import DefaultEmbeddingFunction + from .sources import get_tutorial_sources + from .sources import HOME_SOURCES, HELP_SOURCES, TUTORIAL_SOURCES + from .sources import READTHEDOCS_SOURCES, BOOK_HTML_SOURCES + + print("Loading embedding model (ONNX all-MiniLM-L6-v2)...") + model = DefaultEmbeddingFunction() + + print(f"ChromaDB path: {get_chroma_path()}") + collection = get_collection(reset=args.reset) + + print("Fetching tutorial list from GSAS-II repository…") + tutorial_sources = get_tutorial_sources() + print(f" {len(tutorial_sources)} tutorials found. ({len(TUTORIAL_SOURCES)} in hard-coded list)") + + total_chunks = 0 + + # add a notes to distinguish information sources + # W: Web; T: Tutorial; M: Manual; (Help pages & Book already tagged.) + for i in HOME_SOURCES: + if 'title' in i: + i['title'] += ' (W)' + + for i in tutorial_sources: + if 'title' in i: + i['title'] += ' (T)' + + html_sources = [HOME_SOURCES, HELP_SOURCES, tutorial_sources] + if args.manual: + for i in READTHEDOCS_SOURCES: + if 'title' in i: + i['title'] += ' (M)' + + print(f" Adding Programmers' manual ({len(READTHEDOCS_SOURCES)} HTML pages)") + html_sources = html_sources + [READTHEDOCS_SOURCES] + if args.book: + html_sources = html_sources + [BOOK_HTML_SOURCES] + + print("\n=== Ingesting HTML pages ===") + for pagelist in html_sources: + for source in pagelist: + if type(source) is str: + print(f"\n*** processing {source}") + continue + total_chunks += ingest_html_source(source, collection, model) + if not args.html_only: + print("\n=== Ingesting PDFs ===") + from .sources import PDF_SOURCES + for source in PDF_SOURCES: + total_chunks += ingest_pdf_source(source, collection, model) + + print(f"\nDone. Total chunks in collection: {collection.count()}") + + +if __name__ == "__main__": + multiprocessing.freeze_support() + main() diff --git a/GSASII/gsas_query/meson.build b/GSASII/gsas_query/meson.build new file mode 100644 index 00000000..f85b43fe --- /dev/null +++ b/GSASII/gsas_query/meson.build @@ -0,0 +1,15 @@ +py.install_sources([ + '__init__.py', + '_paths.py', + '_web.py', + 'app.py', + 'cli.py', + 'gui.py', + 'ingest.py', + 'rag.py', + 'sources.py', + 'wordcount.py', +], + pure: false, # Will be installed next to binaries + subdir: 'GSASII/gsas_query' # Folder relative to site-packages to install to +) diff --git a/GSASII/gsas_query/rag.py b/GSASII/gsas_query/rag.py new file mode 100644 index 00000000..07485bba --- /dev/null +++ b/GSASII/gsas_query/rag.py @@ -0,0 +1,377 @@ +""" +RAG engine: query ChromaDB, retrieve relevant chunks, generate answer. + +Backend selection (in priority order): + 1. If ``LLM_BACKEND`` env var is set explicitly, it is always honoured. + 2. If ``LLM_BACKEND`` is not set and ``llama_cpp`` (llama-cpp-python) is + importable, the llama_cpp backend is selected automatically. + 3. Otherwise the default is ``ollama``. + +Supported LLM_BACKEND values: + "ollama" — local Ollama server, fully on-premises + "anthropic" — Anthropic Claude API (requires ANTHROPIC_API_KEY) + "llama_cpp" — in-process llama-cpp-python (requires LLAMA_CPP_MODEL path) + "retrieval" — no LLM; returns raw matched chunks (offline / testing) +""" + +import os +from functools import lru_cache + +import chromadb +from chromadb.utils.embedding_functions import DefaultEmbeddingFunction + +from ._paths import get_chroma_path + +COLLECTION_NAME = "gsasii_docs" +TOP_K = 6 + +SYSTEM_PROMPT = """\ +You are an expert assistant for GSAS-II (General Structure Analysis System-II), \ +crystallographic analysis software developed at Argonne National Laboratory. + +You answer questions using the GSAS-II tutorials, help manual, and documentation \ +provided as context. Your users are crystallographers, materials scientists, and \ +physicists who need precise, actionable answers. + +Guidelines: +- Be specific and technical. Use proper crystallographic terminology. +- When a question involves a step-by-step procedure, preserve the numbered steps. +- Each context section is numbered [1], [2], etc. Cite sources inline as you write \ + by inserting the number in brackets (e.g. "Open the Phase tab [1] and select..."). \ + Place the citation immediately after the sentence or clause it supports. Do NOT add \ + a separate references section at the end. Do NOT reproduce the raw "[Source: ...]" \ + labels from the context — use only the numeric [N] markers. +- If the provided context does not contain enough information to answer, say so clearly \ + and suggest which tutorial might cover the topic. +- Do not fabricate parameter names, menu paths, or file formats. +""" + + +@lru_cache(maxsize=1) +def _get_ef() -> DefaultEmbeddingFunction: + return DefaultEmbeddingFunction() + + +@lru_cache(maxsize=1) +def _get_collection() -> chromadb.Collection: + client = chromadb.PersistentClient(path=get_chroma_path()) + return client.get_or_create_collection(COLLECTION_NAME) + + +def _retrieve(question: str) -> tuple[str, list[dict], dict[str, dict], list[str]]: + collection = _get_collection() + + if collection.count() == 0: + return "", [], {}, [] + + embedding = _get_ef()([question])[0] + results = collection.query( + query_embeddings=[embedding], + n_results=TOP_K, + include=["documents", "metadatas", "distances"], + ) + + context_parts = [] + citations: dict[str, dict] = {} + sources = [] + seen_sources: set = set() + chunk_texts: list[str] = [] + + # Normalise distances within the result set so the best match = 100% + # and others are proportional. Raw cosine distances from all-MiniLM tend + # to cluster near 1.0 even for good matches, making raw % look misleading. + distances = results["distances"][0] + min_d = min(distances) if distances else 0.0 + max_d = max(distances) if distances else 1.0 + span = (max_d - min_d) or 1.0 + + for i, (doc, meta, dist) in enumerate(zip( + results["documents"][0], + results["metadatas"][0], + distances, + ), start=1): + context_parts.append( + f"[{i}] [Source: {meta['title']} | Section: {meta['section']}]\n{doc}" + ) + relevance = round((max_d - dist) / span, 3) # 1.0 = best, 0.0 = weakest + citations[str(i)] = { + "title": meta["title"], + "section": meta["section"], + "url": meta["url"], + "relevance": relevance, + } + chunk_texts.append(doc) + source_key = (meta["url"], meta["section"]) + if source_key not in seen_sources: + seen_sources.add(source_key) + sources.append({ + "title": meta["title"], + "section": meta["section"], + "url": meta["url"], + "category": meta.get("category", ""), + "relevance": relevance, + }) + + context = "\n\n---\n\n".join(context_parts) + return context, sources, citations, chunk_texts + + +def _build_messages(question: str, context: str, history: list[dict]) -> list[dict]: + messages = [] + for turn in history[-6:]: + if turn.get("role") in {"user", "assistant"}: + messages.append({"role": turn["role"], "content": turn["content"]}) + + user_content = ( + f"Context from GSAS-II documentation:\n\n{context}\n\n" + f"---\n\nQuestion: {question}" + if context + else question + ) + messages.append({"role": "user", "content": user_content}) + return messages + + +def _effective_backend() -> str: + """Return the backend to use. + + Priority: + 1. ``LLM_BACKEND`` env var when explicitly set. + 2. ``llama_cpp`` when llama-cpp-python is importable and LLM_BACKEND unset. + 3. ``ollama`` as the final default. + """ + env_backend = os.environ.get("LLM_BACKEND", "").strip().lower() + if env_backend: + return env_backend + try: + import llama_cpp # noqa: F401 + return "llama_cpp" + except ImportError: + pass + return "ollama" + + +def _answer_anthropic(messages: list[dict]) -> str: + import anthropic + client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"]) + response = client.messages.create( + model=os.environ.get("ANTHROPIC_MODEL", "claude-sonnet-4-6"), + max_tokens=1500, + system=SYSTEM_PROMPT, + messages=messages, + ) + return response.content[0].text + + +@lru_cache(maxsize=1) +def _get_llama(model_path: str, n_ctx: int): + from llama_cpp import Llama + return Llama(model_path=model_path, n_ctx=n_ctx, verbose=False) + + +def _answer_llama_cpp(messages: list[dict]) -> str: + model_path = os.environ.get("LLAMA_CPP_MODEL", "").strip() + if not model_path: + raise RuntimeError( + "LLAMA_CPP_MODEL is not set. " + "Set it to the path of a GGUF model file, e.g. " + "LLAMA_CPP_MODEL=/path/to/model.gguf" + ) + n_ctx = int(os.environ.get("LLAMA_CPP_N_CTX", "4096")) + # Cap at 800 tokens for small models to prevent repetition loops. + max_tokens = int(os.environ.get("LLAMA_CPP_MAX_TOKENS", "800")) + llm = _get_llama(model_path, n_ctx) + full_messages = [{"role": "system", "content": SYSTEM_PROMPT}] + messages + response = llm.create_chat_completion(messages=full_messages, max_tokens=max_tokens) + return response["choices"][0]["message"]["content"] + + +def _ollama_url() -> str: + return os.environ.get("OLLAMA_URL", "http://localhost:11434") + + +def _installed_ollama_models() -> list[str]: + import httpx + resp = httpx.get(f"{_ollama_url()}/api/tags", timeout=5) + resp.raise_for_status() + data = resp.json() or {} + return [m.get("name", "") for m in data.get("models", []) if m.get("name")] + + +def _choose_ollama_model() -> str: + preferred = os.environ.get("OLLAMA_MODEL", "").strip() + models = _installed_ollama_models() + + if not models: + raise RuntimeError( + "No Ollama models are installed. Run e.g. " + "`ollama pull llama3.1:8b` or `ollama pull qwen2.5:3b`." + ) + + if preferred: + if preferred in models: + return preferred + raise RuntimeError( + f"OLLAMA_MODEL='{preferred}' is not installed. " + f"Available models: {', '.join(models)}" + ) + + for candidate in ("llama3.1:8b", "llama3", "qwen2.5:3b"): + if candidate in models: + return candidate + + return models[0] + + +def _answer_ollama(messages: list[dict]) -> str: + import httpx + ollama_url = _ollama_url() + ollama_model = _choose_ollama_model() + + full_messages = [{"role": "system", "content": SYSTEM_PROMPT}] + messages + try: + resp = httpx.post( + f"{ollama_url}/api/chat", + json={"model": ollama_model, "messages": full_messages, "stream": False}, + timeout=120, + ) + resp.raise_for_status() + except httpx.HTTPStatusError as e: + detail = "" + try: + detail = f" | Response: {e.response.text}" + except Exception: + pass + raise RuntimeError(f"Ollama API error: {e}{detail}") from e + return resp.json()["message"]["content"] + + +def answer_question(question: str, history: list[dict]) -> dict: + """Main entry point: returns {answer, sources, citations, backend}.""" + if not question.strip(): + return {"answer": "Please enter a question.", "sources": [], "citations": {}, "backend": ""} + + context, sources, citations, chunk_texts = _retrieve(question) + + if not context: + return { + "answer": ( + "The knowledge base is empty. Run `gsas-query --setup` " + "to index the GSAS-II documentation." + ), + "sources": [], + "citations": {}, + "backend": "", + } + + backend = _effective_backend() + + if backend == "retrieval": + answer = ( + "Most relevant sections (no LLM synthesis — install llama-cpp-python, " + "Ollama, or set LLM_BACKEND=anthropic for generated answers):\n\n" + context + ) + return {"answer": answer, "sources": sources, "citations": citations, "backend": backend} + + messages = _build_messages(question, context, history) + + if backend == "llama_cpp": + answer = _answer_llama_cpp(messages) + elif backend == "ollama": + answer = _answer_ollama(messages) + else: + answer = _answer_anthropic(messages) + + answer = _inject_citations(answer, chunk_texts) + answer, citations = _renumber_by_appearance(answer, citations) + return {"answer": answer, "sources": sources, "citations": citations, "backend": backend} + + +def _renumber_by_appearance(answer: str, citations: dict) -> tuple[str, dict]: + """Renumber [N] markers so citations are sequential in order of first appearance. + + The injector assigns numbers by relevance rank, so [4] may appear before [2] + in the text. This makes citations appear in order: first cited = [1], etc. + Sources not cited inline are appended at the end of the new citations dict + (for the bibliography) but get no inline marker. + """ + import re + + # Walk the text to get first-appearance order of each cited number + appeared: list[str] = [] + seen: set[str] = set() + for m in re.finditer(r'\[(\d+)\]', answer): + k = m.group(1) + if k in citations and k not in seen: + appeared.append(k) + seen.add(k) + + # Append any retrieved-but-uncited chunks (for full bibliography) + for k in sorted(citations, key=int): + if k not in seen: + appeared.append(k) + + # old key → new sequential key + old_to_new = {old: str(i + 1) for i, old in enumerate(appeared)} + + # Rewrite [N] markers in the answer text + new_answer = re.sub( + r'\[(\d+)\]', + lambda m: f'[{old_to_new.get(m.group(1), m.group(1))}]', + answer, + ) + + # Rebuild citations dict in new order + new_citations = {old_to_new[k]: citations[k] for k in appeared} + return new_answer, new_citations + + +def _inject_citations(answer: str, chunk_texts: list[str]) -> str: + """Inject [N] citation markers inline using keyword overlap with context chunks. + + Small models (3B) rarely cite more than one source. This post-processor + scans each substantive line, finds the best-matching context chunk by + content-word overlap, and appends [N] if not already present. Requires at + least 4 shared content words (≥4 chars) to avoid spurious matches. + """ + import re + + # Pre-compute content-word sets for each chunk (1-indexed) + _STOP = {"gsas", "gsasii", "that", "with", "this", "from", "will", "have", + "your", "then", "they", "each", "also", "been", "used", "when"} + + def content_words(text: str) -> set[str]: + return {w for w in re.findall(r'\b[a-zA-Z]{4,}\b', text.lower()) + if w not in _STOP} + + chunk_word_sets = [content_words(t) for t in chunk_texts] + + def best_chunk_for(line: str) -> int | None: + lwords = content_words(line) + if len(lwords) < 5: + return None + best_score = 3 # minimum overlap threshold + best_n = None + for n, cwords in enumerate(chunk_word_sets, start=1): + score = len(lwords & cwords) + if score > best_score: + best_score = score + best_n = n + return best_n + + lines = answer.split('\n') + result = [] + for line in lines: + stripped = line.strip() + # Skip: already cited, blank, bullet markers alone, or very short + if (not stripped + or re.search(r'\[\d+\]', stripped) + or len(stripped) < 40): + result.append(line) + continue + n = best_chunk_for(stripped) + if n is not None: + result.append(line.rstrip() + f' [{n}]') + else: + result.append(line) + return '\n'.join(result) diff --git a/GSASII/gsas_query/sources.py b/GSASII/gsas_query/sources.py new file mode 100644 index 00000000..240bef19 --- /dev/null +++ b/GSASII/gsas_query/sources.py @@ -0,0 +1,474 @@ +""" +Complete list of GSAS-II documentation sources. + +All Home page, Help & Tutorial URLs resolve under: + https://advancedphotonsource.github.io/GSAS-II-tutorials/ + URL lists in HOME_SOURCES, HELP_SOURCES and TUTORIAL_SOURCES + +Additional HTML sources: + - GSAS-II Programmer's Guide — 24 pages from readthedocs (https://gsas-ii.readthedocs.io/en/latest) + URL list in READTHEDOCS_SOURCES + + - Powder Diffraction Crystallography book — 185 HTML pages (https://briantoby.github.io/PowderCrystallography) + URL list in BOOK_HTML_SOURCES + +Sources are selected based on the arguments supplied when ingest.py is run (e.g. --book includes BOOK_HTML_SOURCES) + +""" + +BASE_URL = "https://advancedphotonsource.github.io/GSAS-II-tutorials" + +# ── Home / installation pages (22) ──────────────────────────────────────────── + +HOME_SOURCES = [ + "GSAS-II Home pages", + {"title": "GSAS-II Home", "url": f"{BASE_URL}/index.html", "category": "Home"}, + {"title": "About GSAS-II", "url": f"{BASE_URL}/AboutGSASII.html", "category": "Home"}, + {"title": "Documentation Overview", "url": f"{BASE_URL}/documentation.html", "category": "Home"}, + {"title": "Tutorials Index", "url": f"{BASE_URL}/tutorials.html", "category": "Home"}, + {"title": "Help Overview", "url": f"{BASE_URL}/help.html", "category": "Home"}, + {"title": "Miscellaneous Notes", "url": f"{BASE_URL}/misc.html", "category": "Home"}, + {"title": "Installation Overview", "url": f"{BASE_URL}/install.html", "category": "Installation"}, + {"title": "Install with pip", "url": f"{BASE_URL}/install-pip.html", "category": "Installation"}, + {"title": "Install with pixi", "url": f"{BASE_URL}/install_pixi.html", "category": "Installation"}, + {"title": "Install GSAS2Full (Mac)", "url": f"{BASE_URL}/install-g2f-mac.html", "category": "Installation"}, + {"title": "Install GSAS2Full (Linux)", "url": f"{BASE_URL}/install-g2f-linux.html", "category": "Installation"}, + {"title": "Install GSAS2Full (Windows)", "url": f"{BASE_URL}/install-g2f-win.html", "category": "Installation"}, + {"title": "Install for Developers", "url": f"{BASE_URL}/install_dev.html", "category": "Installation"}, + {"title": "Install External Programs", "url": f"{BASE_URL}/install-external.html", "category": "Installation"}, + {"title": "Install Python Manually", "url": f"{BASE_URL}/install-python.html", "category": "Installation"}, + {"title": "GSAS-II Options", "url": f"{BASE_URL}/options.html", "category": "Home"}, + {"title": "Proxy Configuration", "url": f"{BASE_URL}/proxy.html", "category": "Installation"}, + {"title": "Developer Notes", "url": f"{BASE_URL}/developers.html", "category": "Development"}, + {"title": "Compiling Extensions", "url": f"{BASE_URL}/compile.html", "category": "Development"}, + {"title": "Mailing List", "url": f"{BASE_URL}/mailinglist.html", "category": "Home"}, + {"title": "Bug Reporting", "url": f"{BASE_URL}/bug.html", "category": "Home"}, + {"title": "General Index", "url": f"{BASE_URL}/genindex.html", "category": "Home"}, +] + +# ── Help pages (42, skipping 404.html) ──────────────────────────────────────── + +HELP_SOURCES = [ + "GSAS-II Help pages", + {"title": "Help: Application Window", "url": f"{BASE_URL}/help/applicationwindow.html", "category": "Help"}, + {"title": "Help: Main Menu", "url": f"{BASE_URL}/help/mainmenu.html", "category": "Help"}, + {"title": "Help: Data Tree", "url": f"{BASE_URL}/help/datatree.html", "category": "Help"}, + {"title": "Help: Common Tree Items", "url": f"{BASE_URL}/help/commontreeitems.html", "category": "Help"}, + {"title": "Help: Preface", "url": f"{BASE_URL}/help/preface.html", "category": "Help"}, + {"title": "Help: Others", "url": f"{BASE_URL}/help/others.html", "category": "Help"}, + {"title": "Help: Powder Diffraction Overview", "url": f"{BASE_URL}/help/powder.html", "category": "Help: Powder"}, + {"title": "Help: Powder Parent", "url": f"{BASE_URL}/help/powderparent.html", "category": "Help: Powder"}, + {"title": "Help: Powder Comments", "url": f"{BASE_URL}/help/powdercomments.html", "category": "Help: Powder"}, + {"title": "Help: Powder Instrument Parameters", "url": f"{BASE_URL}/help/powderinst.html", "category": "Help: Powder"}, + {"title": "Help: Powder Sample Parameters", "url": f"{BASE_URL}/help/powdersample.html", "category": "Help: Powder"}, + {"title": "Help: Powder Limits", "url": f"{BASE_URL}/help/powderlimits.html", "category": "Help: Powder"}, + {"title": "Help: Powder Background", "url": f"{BASE_URL}/help/powderbkg.html", "category": "Help: Powder"}, + {"title": "Help: Powder Peaks", "url": f"{BASE_URL}/help/powderpeaks.html", "category": "Help: Powder"}, + {"title": "Help: Powder Peak Indexing", "url": f"{BASE_URL}/help/powderindexppeaks.html", "category": "Help: Powder"}, + {"title": "Help: Powder Cells", "url": f"{BASE_URL}/help/powdercells.html", "category": "Help: Powder"}, + {"title": "Help: Powder Reflections", "url": f"{BASE_URL}/help/powderrefs.html", "category": "Help: Powder"}, + {"title": "Help: Peak List", "url": f"{BASE_URL}/help/peaks.html", "category": "Help: Powder"}, + {"title": "Help: Sequential Refinement", "url": f"{BASE_URL}/help/sequential.html", "category": "Help: Powder"}, + {"title": "Help: Phase Overview", "url": f"{BASE_URL}/help/phaseoverview.html", "category": "Help: Phase"}, + {"title": "Help: Phase General", "url": f"{BASE_URL}/help/phasegeneral.html", "category": "Help: Phase"}, + {"title": "Help: Phase Atoms", "url": f"{BASE_URL}/help/phaseatoms.html", "category": "Help: Phase"}, + {"title": "Help: Phase Data", "url": f"{BASE_URL}/help/phasedata.html", "category": "Help: Phase"}, + {"title": "Help: Phase Draw Options", "url": f"{BASE_URL}/help/phasedrawopts.html", "category": "Help: Phase"}, + {"title": "Help: Phase Draw Atoms", "url": f"{BASE_URL}/help/phasedrawatoms.html", "category": "Help: Phase"}, + {"title": "Help: Phase Map Peaks", "url": f"{BASE_URL}/help/phasemappeaks.html", "category": "Help: Phase"}, + {"title": "Help: Phase Pawley", "url": f"{BASE_URL}/help/phasepawley.html", "category": "Help: Phase"}, + {"title": "Help: Phase Texture", "url": f"{BASE_URL}/help/phasetexture.html", "category": "Help: Phase"}, + {"title": "Help: Phase Layers (Stacking)", "url": f"{BASE_URL}/help/phaselayers.html", "category": "Help: Phase"}, + {"title": "Help: Phase Waves (Modulated)", "url": f"{BASE_URL}/help/phasewave.html", "category": "Help: Phase"}, + {"title": "Help: Phase Rigid Bodies", "url": f"{BASE_URL}/help/phaseRB.html", "category": "Help: Phase"}, + {"title": "Help: Phase RMC", "url": f"{BASE_URL}/help/phaseRMC.html", "category": "Help: Phase"}, + {"title": "Help: Phase MCSA", "url": f"{BASE_URL}/help/phasemcsa.html", "category": "Help: Phase"}, + {"title": "Help: Phase ISODISTORT", "url": f"{BASE_URL}/help/phaseisodistort.html", "category": "Help: Phase"}, + {"title": "Help: Phase DYSNOMIA", "url": f"{BASE_URL}/help/phasedysnomia.html", "category": "Help: Phase"}, + {"title": "Help: Single Crystal", "url": f"{BASE_URL}/help/singlecrystal.html", "category": "Help: Single Crystal"}, + {"title": "Help: Image Processing", "url": f"{BASE_URL}/help/image.html", "category": "Help: Image"}, + {"title": "Help: Small Angle Scattering", "url": f"{BASE_URL}/help/smallanglescattering.html","category": "Help: SAXS"}, + {"title": "Help: Pair Distribution Function", "url": f"{BASE_URL}/help/pairdistribution.html", "category": "Help: PDF"}, + {"title": "Help: Reflectometry", "url": f"{BASE_URL}/help/reflectometry.html", "category": "Help: Reflectometry"}, + {"title": "Help: Cluster Analysis", "url": f"{BASE_URL}/help/cluster.html", "category": "Help: Analysis"}, + {"title": "Help: Index", "url": f"{BASE_URL}/help/index.html", "category": "Help"}, +] + +# ── Tutorials (62, skipping tutorial_template) ──────────────────────────────── + +TUTORIAL_SOURCES = [ + "GSAS-II Tutorials", + # Getting Started + {"title": "Starting GSAS-II", + "url": f"{BASE_URL}/StartingGSASII/Starting%20GSAS.htm", + "category": "Getting Started"}, + + # Rietveld Refinement + {"title": "Fitting CW Neutron Powder Data (YIG)", + "url": f"{BASE_URL}/CWNeutron/Neutron%20CW%20Powder%20Data.htm", + "category": "Rietveld Refinement"}, + {"title": "Fitting Laboratory X-ray Powder Data (Fluoroapatite)", + "url": f"{BASE_URL}/LabData/Laboratory%20X.htm", + "category": "Rietveld Refinement"}, + {"title": "Combined X-ray and CW-Neutron Refinement (PbSO4)", + "url": f"{BASE_URL}/CWCombined/Combined%20refinement.htm", + "category": "Rietveld Refinement"}, + {"title": "Combined X-ray and TOF-Neutron Rietveld Refinement", + "url": f"{BASE_URL}/TOF-CW%20Joint%20Refinement/TOF%20combined%20XN%20Rietveld%20refinement%20in%20GSAS.htm", + "category": "Rietveld Refinement"}, + {"title": "Simulating Powder Diffraction with GSAS-II", + "url": f"{BASE_URL}/Simulation/SimTutorial.htm", + "category": "Rietveld Refinement"}, + + # Background and Profile + {"title": "Fitting the Background using Fixed Points", + "url": f"{BASE_URL}/BkgFit/FitBkgTut.htm", + "category": "Background & Profile"}, + {"title": "Using the Auto Background Feature", + "url": f"{BASE_URL}/AutoBkg/AutoBkg.html", + "category": "Background & Profile"}, + {"title": "Le Bail Intensity Extraction (Sucrose)", + "url": f"{BASE_URL}/LeBail/LeBailSucrose.htm", + "category": "Background & Profile"}, + {"title": "Determining Profile Parameters with Fundamental Parameters", + "url": f"{BASE_URL}/FPAfit/FPAfit.htm", + "category": "Background & Profile"}, + {"title": "Create Instrument Parameter File (CW Profile from Standard)", + "url": f"{BASE_URL}/CWInstDemo/FindProfParamCW.html", + "category": "Background & Profile"}, + {"title": "Use of Parameter Limits", + "url": f"{BASE_URL}/ParameterLimits/ParameterLimitsUse.html", + "category": "Background & Profile"}, + {"title": "Rietveld Fitting with Rigid Bodies", + "url": f"{BASE_URL}/RigidBody/RigidBodyRef.html", + "category": "Background & Profile"}, + + # Sequential Refinement + {"title": "Sequential Refinement of Multiple Datasets", + "url": f"{BASE_URL}/SeqRefine/SequentialTutorial.htm", + "category": "Sequential Refinement"}, + {"title": "Parametric Fitting and Pseudo Variables for Sequential Fits", + "url": f"{BASE_URL}/SeqParametric/ParametricFitting.htm", + "category": "Sequential Refinement"}, + {"title": "Sequential Fitting of Single Peaks and Strain Analysis", + "url": f"{BASE_URL}/TOF%20Sequential%20Single%20Peak%20Fit/TOF%20Sequential%20Single%20Peak%20Fit.htm", + "category": "Sequential Refinement"}, + {"title": "Sequential Refinement with Small Angle Scattering Data", + "url": f"{BASE_URL}/SAseqref/Sequential%20Refinement%20of%20Small%20Angle%20Scattering%20Data.htm", + "category": "Sequential Refinement"}, + + # Magnetic Structures + {"title": "Simple Magnetic Structure Analysis", + "url": f"{BASE_URL}/SimpleMagnetic/SimpleMagnetic.htm", + "category": "Magnetic Structures"}, + {"title": "Register for Bilbao Crystallographic Server", + "url": f"{BASE_URL}/RegisterBilbao/RegisterBilbao.html", + "category": "Magnetic Structures"}, + {"title": "Magnetic Structure Analysis I", + "url": f"{BASE_URL}/Magnetic-I/Magnetic%20Structures-I.htm", + "category": "Magnetic Structures"}, + {"title": "Magnetic Structure Analysis II", + "url": f"{BASE_URL}/Magnetic-II/Magnetic-II.htm", + "category": "Magnetic Structures"}, + {"title": "Magnetic Structure Analysis III", + "url": f"{BASE_URL}/Magnetic-III/Magnetic-III.htm", + "category": "Magnetic Structures"}, + {"title": "Magnetic Structure Analysis IV", + "url": f"{BASE_URL}/Magnetic-IV/Magnetic-IV.htm", + "category": "Magnetic Structures"}, + {"title": "Magnetic Structure Analysis V", + "url": f"{BASE_URL}/Magnetic-V/Magnetic-V.htm", + "category": "Magnetic Structures"}, + {"title": "k-vector Searching in GSAS-II (zero vector)", + "url": f"{BASE_URL}/k_vec_tutorial/k_vec_tutorial.html", + "category": "Magnetic Structures"}, + {"title": "k-vector Searching in GSAS-II (non-zero vector)", + "url": f"{BASE_URL}/k_vec_tutorial_non_zero/k_vec_tutorial_non_zero.html", + "category": "Magnetic Structures"}, + {"title": "Use of ISODISTORT with k-vector from GSAS-II", + "url": f"{BASE_URL}/k_vec_isodistort/k_vec_isodistort.html", + "category": "Magnetic Structures"}, + + # Structure Solution + {"title": "Fitting Individual Peaks and Autoindexing", + "url": f"{BASE_URL}/FitPeaks/Fit%20Peaks.htm", + "category": "Structure Solution"}, + {"title": "Charge Flipping Structure Solution (Jadarite)", + "url": f"{BASE_URL}/CFjadarite/Charge%20Flipping%20in%20GSAS.htm", + "category": "Structure Solution"}, + {"title": "Charge Flipping Structure Solution (Sucrose)", + "url": f"{BASE_URL}/CFsucrose/Charge%20Flipping%20-%20sucrose.htm", + "category": "Structure Solution"}, + {"title": "Charge Flipping from X-ray Single Crystal Data", + "url": f"{BASE_URL}/CFXraySingleCrystal/CFSingleCrystal.htm", + "category": "Structure Solution"}, + {"title": "Charge Flipping from Neutron TOF Single Crystal Data", + "url": f"{BASE_URL}/TOF%20Charge%20Flipping/Charge%20Flipping%20with%20TOF%20single%20crystal%20data%20in%20GSASII.htm", + "category": "Structure Solution"}, + {"title": "Monte-Carlo Simulated Annealing Structure Determination", + "url": f"{BASE_URL}/MCsimanneal/MCSA%20in%20GSAS.htm", + "category": "Structure Solution"}, + {"title": "Merohedral Twin Refinements", + "url": f"{BASE_URL}/MerohedralTwins/Merohedral%20twin%20refinement%20in%20GSAS.htm", + "category": "Structure Solution"}, + {"title": "Single Crystal Refinement from TOF Data", + "url": f"{BASE_URL}/TOF%20Single%20Crystal%20Refinement/TOF%20single%20crystal%20refinement%20in%20GSAS.htm", + "category": "Structure Solution"}, + + # PDF: RMCProfile + {"title": "RMC Modeling with RMCProfile I", + "url": f"{BASE_URL}/RMCProfile-I/RMCProfile-I.htm", + "category": "PDF: RMCProfile"}, + {"title": "RMC Modeling with RMCProfile II", + "url": f"{BASE_URL}/RMCProfile-II/RMCProfile-II.htm", + "category": "PDF: RMCProfile"}, + {"title": "RMC Modeling with RMCProfile III", + "url": f"{BASE_URL}/RMCProfile-III/RMCProfile-III.htm", + "category": "PDF: RMCProfile"}, + {"title": "RMC Modeling with RMCProfile IV", + "url": f"{BASE_URL}/RMCProfile-IV/RMCProfile-IV.htm", + "category": "PDF: RMCProfile"}, + + # PDF: PDFfit + {"title": "Small Box PDF Modeling with PDFfit I", + "url": f"{BASE_URL}/PDFfit-I/PDFfit-I.htm", + "category": "PDF: PDFfit"}, + {"title": "Small Box PDF Modeling with PDFfit II", + "url": f"{BASE_URL}/PDFfit-II/PDFfit-II.htm", + "category": "PDF: PDFfit"}, + {"title": "Sequential PDF Fitting with PDFfit III", + "url": f"{BASE_URL}/PDFfit-III/PDFfit-III.htm", + "category": "PDF: PDFfit"}, + {"title": "Nanoparticle PDF Fitting with PDFfit IV", + "url": f"{BASE_URL}/PDFfit-IV/PDFfit-IV.htm", + "category": "PDF: PDFfit"}, + + # PDF: fullrmc + {"title": "RMC and Rigid Body Modeling with fullrmc (Ni)", + "url": f"{BASE_URL}/fullrmc-Ni/fullrmc-Ni.html", + "category": "PDF: fullrmc"}, + {"title": "RMC and Rigid Body Modeling with fullrmc (SF6)", + "url": f"{BASE_URL}/fullrmc-SF6/fullrmc-SF6.html", + "category": "PDF: fullrmc"}, + + # Stacking Faults + {"title": "Stacking Fault Simulations (Diamond)", + "url": f"{BASE_URL}/StackingFaults-I/Stacking%20Faults-I.htm", + "category": "Stacking Faults"}, + {"title": "Stacking Fault Simulations (Keokuk Kaolinite)", + "url": f"{BASE_URL}/StackingFaults-II/Stacking%20Faults%20II.htm", + "category": "Stacking Faults"}, + {"title": "Stacking Fault Simulations (Georgia Kaolinite)", + "url": f"{BASE_URL}/StackingFaults-III/Stacking%20Faults-III.htm", + "category": "Stacking Faults"}, + + # TOF Calibration + {"title": "Calibration of a Neutron TOF Diffractometer", + "url": f"{BASE_URL}/TOF%20Calibration/Calibration%20of%20a%20TOF%20powder%20diffractometer.htm", + "category": "Calibration"}, + + # 2D Image Processing + {"title": "Calibration of an Area Detector", + "url": f"{BASE_URL}/2DCalibration/Calibration%20of%20an%20area%20detector%20in%20GSAS.htm", + "category": "2D Image Processing"}, + {"title": "Integration of Area Detector Data", + "url": f"{BASE_URL}/2DIntegration/Integration%20of%20area%20detector%20data%20in%20GSAS.htm", + "category": "2D Image Processing"}, + {"title": "Strain Fitting of 2D Data", + "url": f"{BASE_URL}/2DStrain/Strain%20fitting%20of%202D%20data%20in%20GSAS-II.htm", + "category": "2D Image Processing"}, + {"title": "Texture Analysis of 2D Data", + "url": f"{BASE_URL}/2DTexture/Texture%20analysis%20of%202D%20data%20in%20GSAS-II.htm", + "category": "2D Image Processing"}, + {"title": "Area Detector Calibration: Determine Wavelength", + "url": f"{BASE_URL}/DeterminingWavelength/DeterminingWavelength.html", + "category": "2D Image Processing"}, + {"title": "Area Detector Calibration: Detector Distances", + "url": f"{BASE_URL}/CalibrationTutorial/CalibrationTutorial.html", + "category": "2D Image Processing"}, + + # Small Angle Scattering + {"title": "Small Angle X-ray Data Size Distribution", + "url": f"{BASE_URL}/SAsize/Small%20Angle%20Size%20Distribution.htm", + "category": "Small Angle Scattering"}, + {"title": "Fitting Small Angle X-ray Data", + "url": f"{BASE_URL}/SAfit/Fitting%20Small%20Angle%20Scattering%20Data.htm", + "category": "Small Angle Scattering"}, + {"title": "Image Processing of Small Angle X-ray Data", + "url": f"{BASE_URL}/SAimages/Small%20Angle%20Image%20Processing.htm", + "category": "Small Angle Scattering"}, + + # Python Scripting + {"title": "Scripting a GSAS-II Refinement from Python", + "url": f"{BASE_URL}/PythonScript/Scripting.htm", + "category": "Python Scripting"}, + {"title": "Running a GSAS-II Refinement from the Command Line", + "url": f"{BASE_URL}/PythonScript/CommandLine.htm", + "category": "Python Scripting"}, + + # Publication + {"title": "Create a CIF for Publication", + "url": f"{BASE_URL}/CIFtutorial/CIFtutorial.html", + "category": "Publication"}, + {"title": "Create a Publication-Ready Rietveld Plot", + "url": f"{BASE_URL}/RietPlot/PublicationPlot.htm", + "category": "Publication"}, + + # Misc + {"title": "Cluster and Outlier Analysis", + "url": f"{BASE_URL}/ClusterAnalysis/Cluster and Outlier Analysis.htm", + "category": "Analysis"}, + {"title": "Changing the GSAS-II Font Size", + "url": f"{BASE_URL}/FontSize/FontSize.html", + "category": "Miscellaneous"}, +] + +# ── GSAS-II Programmer's Guide — readthedocs HTML (preferred over PDF) ──────── + +_RTD = "https://gsas-ii.readthedocs.io/en/latest" + +READTHEDOCS_SOURCES = [ + "GSAS-II programmers' manual", + {"title": "GSAS-II Packages Overview", "url": f"{_RTD}/packages.html", "category": "Programmer's Guide"}, + {"title": "GSAS-II Versioning", "url": f"{_RTD}/versioning.html", "category": "Programmer's Guide"}, + {"title": "Object/Variable Organization", "url": f"{_RTD}/objvarorg.html", "category": "Programmer's Guide"}, + {"title": "GSASII module", "url": f"{_RTD}/GSASII.html", "category": "Programmer's Guide"}, + {"title": "GSASIIobj module", "url": f"{_RTD}/GSASIIobj.html", "category": "Programmer's Guide"}, + {"title": "GSASIIutil module", "url": f"{_RTD}/GSASIIutil.html", "category": "Programmer's Guide"}, + {"title": "GSASIIGUIr module", "url": f"{_RTD}/GSASIIGUIr.html", "category": "Programmer's Guide"}, + {"title": "GSASIIGUI module", "url": f"{_RTD}/GSASIIGUI.html", "category": "Programmer's Guide"}, + {"title": "GSASIIdata module", "url": f"{_RTD}/GSASIIdata.html", "category": "Programmer's Guide"}, + {"title": "GSASIIstruc module", "url": f"{_RTD}/GSASIIstruc.html", "category": "Programmer's Guide"}, + {"title": "GSASIImapvars module", "url": f"{_RTD}/GSASIImapvars.html", "category": "Programmer's Guide"}, + {"title": "GSASIIimage module", "url": f"{_RTD}/GSASIIimage.html", "category": "Programmer's Guide"}, + {"title": "GSASIImath module", "url": f"{_RTD}/GSASIImath.html", "category": "Programmer's Guide"}, + {"title": "GSAS-II Index", "url": f"{_RTD}/GSASIIindex.html", "category": "Programmer's Guide"}, + {"title": "Graphics modules", "url": f"{_RTD}/graphics.html", "category": "Programmer's Guide"}, + {"title": "GSASIIpwd module", "url": f"{_RTD}/GSASIIpwd.html", "category": "Programmer's Guide"}, + {"title": "Small Angle Scattering module", "url": f"{_RTD}/SAS.html", "category": "Programmer's Guide"}, + {"title": "GSASIIscriptable module", "url": f"{_RTD}/GSASIIscriptable.html", "category": "Programmer's Guide"}, + {"title": "GSASIIscripts module", "url": f"{_RTD}/GSASIIscripts.html", "category": "Programmer's Guide"}, + {"title": "GSASIIweb module", "url": f"{_RTD}/GSASIIweb.html", "category": "Programmer's Guide"}, + {"title": "Import modules", "url": f"{_RTD}/imports.html", "category": "Programmer's Guide"}, + {"title": "Export modules", "url": f"{_RTD}/exports.html", "category": "Programmer's Guide"}, + {"title": "G2tools module", "url": f"{_RTD}/G2tools.html", "category": "Programmer's Guide"}, + #{"title": "GSAS-II General Index", "url": f"{_RTD}/indices.html", "category": "Programmer's Guide"}, +] + +# ── PDF sources ──────────────────────────────────────────────────────────────── + +PDF_SOURCES = [] # Book and Programmer's Guide now available as HTML + +# ── Powder Diffraction Crystallography book — 185 HTML pages (Brian Toby) ───── +# https://briantoby.github.io/PowderCrystallography/ +# Pages are accessible by direct URL; include via `gsas-query --setup --book`. + +_BOOK_BASE = "https://briantoby.github.io/PowderCrystallography" + +import requests +def url_exists(url): + try: + # allow_redirects=True ensures 301/302 redirects resolve to the final page + response = requests.head(url, allow_redirects=True, timeout=5) + return response.status_code == 200 + except requests.RequestException: + return False + + +BOOK_HTML_SOURCES = [ + "Powder Diff. Cryst. Book book", + {"title": "Powder Diff. Cryst. Book (Contents)", "url": f"{_BOOK_BASE}/HTML-template.html", "category": "Powder Crystallography Book"}, +] +#for _i in range(1, 7): # these are book section tables of contents, no text +# BOOK_HTML_SOURCES.append({ +# "title": f"Powder Diff. Cryst. Book Part {_i}", +# "url": f"{_BOOK_BASE}/HTML-templatepa{_i}.html", +# "category": "Powder Crystallography Book", +# }) +_i = 0 +lastURL = '' +while True: + _i += 1 +#for _i in range(1, 30): # these are mostly tables of contents & no text, but a few have text + url = f"{_BOOK_BASE}/HTML-templatech{_i}.html" + if url_exists(url): + lastURL = url + BOOK_HTML_SOURCES.append({ + "title": f"Powder Diff. Cryst. Book Chapter {_i}", + "url": url, + "category": "Powder Crystallography Book", + }) + else: + print(f'Last Powder Diff. Cryst. Book Chapter is {_i-1} ({lastURL})') + break + +_i = 0 +lastURL = '' +while True: + _i += 1 +#for _i in range(1, 150): + url = f"{_BOOK_BASE}/HTML-templatese{_i}.html" + if url_exists(url): + lastURL = url + BOOK_HTML_SOURCES.append({ + "title": f"Powder Diff. Cryst. Book Section {_i}", + "url": url, + "category": "Powder Crystallography Book", + }) + else: + print(f'Last Powder Diff. Cryst. Book Section is {_i-1} ({lastURL})') + break + +# ── Dynamic tutorial list from GSAS-II's tutorialIndex.py ──────────────────── +# Fetched at ingest time so new tutorials are picked up automatically. +# Falls back to the hardcoded TUTORIAL_SOURCES on any network or parse error. + +_TUTORIAL_INDEX_URL = ( + "https://raw.githubusercontent.com/AdvancedPhotonSource/GSAS-II" + "/main/GSASII/tutorialIndex.py" +) + + +def get_tutorial_sources(): + """Return tutorial sources from GSAS-II's canonical tutorialIndex.py. + + Uses ast.literal_eval (not eval) so untrusted file content is never executed. + Falls back to the hardcoded TUTORIAL_SOURCES list on any error. + """ + import ast + import urllib.request + + try: + req = urllib.request.Request( + _TUTORIAL_INDEX_URL, headers={"User-Agent": "gsas-query"} + ) + with urllib.request.urlopen(req, timeout=10) as resp: + content = resp.read().decode() + + tree = ast.parse(content) + index_value = None + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "tutorialIndex": + index_value = ast.literal_eval(node.value) + break + if index_value is not None: + break + + if not index_value: + return TUTORIAL_SOURCES + + sources = ['GSAS-II tutorials'] + for entry in index_value: + if len(entry) == 4: + directory, filename, title, _ = entry + sources.append({ + "title": title.strip(), + "url": f"{BASE_URL}/{directory}/{filename}", + "category": "Tutorial", + }) + return sources if sources else TUTORIAL_SOURCES + + except Exception: + return TUTORIAL_SOURCES diff --git a/GSASII/gsas_query/static/index.html b/GSASII/gsas_query/static/index.html new file mode 100644 index 00000000..26d1a988 --- /dev/null +++ b/GSASII/gsas_query/static/index.html @@ -0,0 +1,533 @@ + + + + + +GSAS-II Assistant + + + + +
+ +
+

GSAS-II Assistant

+

Powered by GSAS-II docs

+
+
+
Loading…
+
+ +
+
+
🔬
+

Ask anything about GSAS-II

+

I have read the tutorials, help manual, and documentation. + Ask about Rietveld refinement, sequential fits, structure solution, + scripting, calibration, and more.

+
+ + + + + + +
+
+
+
+ +
+ + +
+
Enter to send  ·  Shift+Enter for new line
+ + + + diff --git a/GSASII/gsas_query/wordcount.py b/GSASII/gsas_query/wordcount.py new file mode 100644 index 00000000..2b6008b7 --- /dev/null +++ b/GSASII/gsas_query/wordcount.py @@ -0,0 +1,78 @@ +""" +word count pipeline: fetch GSAS-II documents and count words +""" + +#import argparse +#import hashlib +#import io +#import multiprocessing +#import re +#import sys +#import time +import requests +from bs4 import BeautifulSoup + +def wordcount_html_source(source: dict): + url = source["url"] + title = source["title"] + category = source["category"] + + print(f" Fetching: {title} ({url})") + try: + response = requests.get(url, timeout=10) + soup = BeautifulSoup(response.text, 'html.parser') + + # Extracts text only from the body, ignoring raw HTML code + text = soup.body.get_text() if soup.body else soup.get_text() + + # Split text by whitespace to count individual words + word_count = len(text.split()) + print(f"{url}: {word_count} words") + return word_count + except Exception as e: + print(f"Could not process {url}: {e}") + return 0 + +def main(): + from .sources import get_tutorial_sources + from .sources import HOME_SOURCES, HELP_SOURCES, TUTORIAL_SOURCES + from .sources import READTHEDOCS_SOURCES, BOOK_HTML_SOURCES + + print("Fetching tutorial list from GSAS-II repository…") + tutorial_sources = get_tutorial_sources() + print(f" {len(tutorial_sources)} tutorials found. ({len(TUTORIAL_SOURCES)} in hard-coded list)") + + html_sources = [HOME_SOURCES, HELP_SOURCES, tutorial_sources] + html_sources = html_sources + [READTHEDOCS_SOURCES] + html_sources = html_sources + [BOOK_HTML_SOURCES] + + print("\n=== Scanning HTML pages ===") + total_words = 0 + wordsDict = {} + filesDict = {} + for pagelist in html_sources: + words = 0 + lbl = '?' + i = 0 + for source in pagelist: + if type(source) is str: + print(f"\n*** processing {source}") + lbl = source + continue + i += 1 + + words += wordcount_html_source(source) + total_words += wordcount_html_source(source) + print(f"words: {words} files: {i}") + wordsDict[lbl] = words + filesDict[lbl] = i + #if i >= 5: break # DEBUG + + print(f"\nDone. Total words: {total_words}") + for lbl in wordsDict: + print(lbl,'words',wordsDict[lbl],'files',filesDict[lbl]) + + +if __name__ == "__main__": +# multiprocessing.freeze_support() + main() diff --git a/docs/source/GSASIIGUIr.rst b/docs/source/GSASIIGUIr.rst index 60d9b328..43276ad0 100644 --- a/docs/source/GSASIIGUIr.rst +++ b/docs/source/GSASIIGUIr.rst @@ -66,6 +66,8 @@ GSAS-II-provided Dialog (full window) routines: Class or function name Description ================================ ================================================================= :func:`G2MessageBox` Displays text typically used for errors or warnings. +:class:`G2ModelessMessage` Displays text similar to :func:`G2MessageBox`but in a + non-modal dialog. :func:`ShowScrolledInfo` Dialog to display longer text where scrolling is possibly needed :class:`gpxFileSelector` File browser dialog for opening existing .gpx files @@ -129,6 +131,7 @@ Class or function name Description :func:`MultipleChoicesSelector` Dialog for displaying fairly complex choices, used for CIF powder histogram imports only :func:`PhaseSelector` Select a phase from a list (used for phase importers) +:func:`LLMsearch` Checks for prerequisites & opens a Query_gsas LLM search window ================================ ================================================================= Miscellaneous GUI support routines: diff --git a/docs/source/packages.rst b/docs/source/packages.rst index 85265eda..c14b30c6 100644 --- a/docs/source/packages.rst +++ b/docs/source/packages.rst @@ -67,16 +67,17 @@ Details for GSAS-II use on these specific platforms follows below: **Raspberry Pi** (ARM) Linux: GSAS-II has been installed on both 32-bit and the 64-bit version of the Raspberry Pi OS (formerly called Raspbian) and some older compiled binaries are provided at present for - both, but 32-bit support may not continue. It is expected that + both. It is expected that these binaries will also function on Ubuntu Linux for Raspberry Pi, - but this has not been tried. + but this has not been tried. Updated 64-bit binaries will be created if someone requests them, + but 32-bit support, probably not. The performance of GSAS-II on a Raspberry Pi is not blindingly fast, but one can indeed run GSAS-II on a motherboard that costs only $15 (perhaps even one that costs $5) and uses <5 Watts! Note that the 64-bit OS is preferred on the models where it can be run (currently including models 3A+, 3B, 3B+, 4, 400, CM3, CM3+, CM4, - and Zero 2 W) . With the 32-bit Raspberry Pi OS, which does run on + and Zero 2 W). With the 32-bit Raspberry Pi OS, which does run on all Raspberry Pi models, it is necessary to use the OS distribution's versions of Python and its packages, `see here for more information `_. @@ -145,9 +146,9 @@ interpreter/package versions: and did not see any problems. * pybaselines: no version issues are known. -For more details on problems noted with specific versions of Python -and Python packages, see comments below and details here: -:attr:`GSASIIdataGUI.versionDict`, +Details on problems noted with specific versions of Python +and Python packages are noted in global `versionDict` in +`GSASIIdataGUI`, see https://github.com/AdvancedPhotonSource/GSAS-II/blob/main/GSASII/GSASIIdataGUI.py. Note that GSAS-II is currently being developed using Python 3.11 through 3.13. @@ -170,9 +171,9 @@ Far fewer packages are required to run GSAS-II on a compute server via the scripting interface and without a GUI. ------------------- - GUI Requirements ------------------- +-------------------------- + GUI Package Requirements +-------------------------- When using the GSAS-II graphical user interface (GUI), the following Python extension packages are required: @@ -241,7 +242,8 @@ optional packages are: the base miniconda and anaconda installations, but if you create an environment for GSAS-II (`conda create -n package-list...`), it will not be added - to that environment unless you request it specifically. + to that environment unless you request it specifically. This is + recommended. * pybaselines: Determines a background for a powder pattern in the "autobackground" option. See https://pybaselines.readthedocs.io and @@ -263,6 +265,30 @@ optional packages are: * sympy: This package performs symbolic computations and is used for k-vector searching with ISODISTORT. +* chromaDB: The chromaDB package is used for LLM (AI-based) + documentation searching. It is needed with both the llama and ollama + backends. ("Help via LLM Docs Search" command.) + +* llama-cpp-python: this provides the llama backend used for LLM + (AI-based) documentation searching. The alternate is to use the + ollama backend for the used for the "Help via LLM Docs Search" + command. + +* huggingface_hub: This is used to install the LLM model used in the + llama backend used for LLM (AI-based) documentation searching. If + the model is downloaded in some other manner (to + ~/.GSASII/llama_models) then huggingface_hub is not needed. + +* ollama: The ollama package in conda and pip does not appear to work, + at least not on all platforms, but can be used as an alternate + backend to llama-cpp for LLM (AI-based) documentation searching with + the "Help via LLM Docs Search" command. + To use ollama rather than llama, you are recommended to install ollama directly from + https://ollama.com/download/. Then either start this as a server + process or define the ``OLLAMA_BIN`` environment variable and + GSAS-II will start the server when the help window is opened and close down the server + process when the window is closed or GSAS-II is ended. + *Conda command*: Should you wish to install Python and the desired packages yourself, this is certainly possible. For Linux, ``apt`` or ``yum`` is an option, as is