feat: Add OrcaRouter as a supported LLM provider

api.py

@@ -29,6 +29,8 @@
 username = os.getenv("NEO4J_USERNAME")
 password = os.getenv("NEO4J_PASSWORD")
 ollama_base_url = os.getenv("OLLAMA_BASE_URL")
+orcarouter_base_url = os.getenv("ORCAROUTER_BASE_URL")
+orcarouter_api_key = os.getenv("ORCAROUTER_API_KEY")
 embedding_model_name = os.getenv("EMBEDDING_MODEL")
 llm_name = os.getenv("LLM")
 # Remapping for Langchain Neo4j integration
@@ -47,7 +49,13 @@
 create_vector_index(neo4j_graph)
 
 llm = load_llm(
-    llm_name, logger=BaseLogger(), config={"ollama_base_url": ollama_base_url}
+    llm_name,
+    logger=BaseLogger(),
+    config={
+        "ollama_base_url": ollama_base_url,
+        "orcarouter_base_url": orcarouter_base_url,
+        "orcarouter_api_key": orcarouter_api_key,
+    },
 )
 
 llm_chain = configure_llm_only_chain(llm)

bot.py

@@ -22,6 +22,8 @@
 username = os.getenv("NEO4J_USERNAME")
 password = os.getenv("NEO4J_PASSWORD")
 ollama_base_url = os.getenv("OLLAMA_BASE_URL")
+orcarouter_base_url = os.getenv("ORCAROUTER_BASE_URL")
+orcarouter_api_key = os.getenv("ORCAROUTER_API_KEY")
 embedding_model_name = os.getenv("EMBEDDING_MODEL")
 llm_name = os.getenv("LLM")
 # Remapping for Langchain Neo4j integration
@@ -49,7 +51,15 @@ def on_llm_new_token(self, token: str, **kwargs) -> None:
         self.container.markdown(self.text)
 
 
-llm = load_llm(llm_name, logger=logger, config={"ollama_base_url": ollama_base_url})
+llm = load_llm(
+    llm_name,
+    logger=logger,
+    config={
+        "ollama_base_url": ollama_base_url,
+        "orcarouter_base_url": orcarouter_base_url,
+        "orcarouter_api_key": orcarouter_api_key,
+    },
+)
 
 llm_chain = configure_llm_only_chain(llm)
 rag_chain = configure_qa_rag_chain(

chains.py

@@ -61,7 +61,19 @@ def load_embedding_model(embedding_model_name: str, logger=BaseLogger(), config=
 
 
 def load_llm(llm_name: str, logger=BaseLogger(), config={}):
-    if llm_name in ["gpt-4", "gpt-4o", "gpt-4-turbo"]:
+    if llm_name.startswith("orcarouter/"):
+        # OrcaRouter exposes its model namespace under the `orcarouter/` prefix on
+        # an OpenAI-compatible endpoint, so it can reuse ChatOpenAI directly.
+        logger.info(f"LLM: Using OrcaRouter: {llm_name}")
+        return ChatOpenAI(
+            temperature=0,
+            model_name=llm_name,
+            streaming=True,
+            base_url=config.get("orcarouter_base_url")
+            or "https://api.orcarouter.ai/v1",
+            api_key=config.get("orcarouter_api_key"),
+        )
+    elif llm_name in ["gpt-4", "gpt-4o", "gpt-4-turbo"]:
         logger.info("LLM: Using GPT-4")
         return ChatOpenAI(temperature=0, model_name=llm_name, streaming=True)
     elif llm_name == "gpt-3.5":

docker-compose.yml

@@ -102,9 +102,11 @@ services:
       - NEO4J_URI=${NEO4J_URI-neo4j://database:7687}
       - NEO4J_PASSWORD=${NEO4J_PASSWORD-password}
       - NEO4J_USERNAME=${NEO4J_USERNAME-neo4j}
-      - OPENAI_API_KEY=${OPENAI_API_KEY-}      
+      - OPENAI_API_KEY=${OPENAI_API_KEY-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY-}
       - OLLAMA_BASE_URL=${OLLAMA_BASE_URL-http://host.docker.internal:11434}
+      - ORCAROUTER_BASE_URL=${ORCAROUTER_BASE_URL-https://api.orcarouter.ai/v1}
+      - ORCAROUTER_API_KEY=${ORCAROUTER_API_KEY-}
       - LLM=${LLM-llama2}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL-sentence_transformer}
       - LANGCHAIN_ENDPOINT=${LANGCHAIN_ENDPOINT-"https://api.smith.langchain.com"}
@@ -144,6 +146,8 @@ services:
       - OPENAI_API_KEY=${OPENAI_API_KEY-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY-}
       - OLLAMA_BASE_URL=${OLLAMA_BASE_URL-http://host.docker.internal:11434}
+      - ORCAROUTER_BASE_URL=${ORCAROUTER_BASE_URL-https://api.orcarouter.ai/v1}
+      - ORCAROUTER_API_KEY=${ORCAROUTER_API_KEY-}
       - LLM=${LLM-llama2}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL-sentence_transformer}
       - LANGCHAIN_ENDPOINT=${LANGCHAIN_ENDPOINT-"https://api.smith.langchain.com"}
@@ -183,8 +187,10 @@ services:
       - NEO4J_PASSWORD=${NEO4J_PASSWORD-password}
       - NEO4J_USERNAME=${NEO4J_USERNAME-neo4j}
       - OPENAI_API_KEY=${OPENAI_API_KEY}
-      - GOOGLE_API_KEY=${GOOGLE_API_KEY}  
+      - GOOGLE_API_KEY=${GOOGLE_API_KEY}
       - OLLAMA_BASE_URL=${OLLAMA_BASE_URL-http://host.docker.internal:11434}
+      - ORCAROUTER_BASE_URL=${ORCAROUTER_BASE_URL-https://api.orcarouter.ai/v1}
+      - ORCAROUTER_API_KEY=${ORCAROUTER_API_KEY-}
       - LLM=${LLM-llama2}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL-sentence_transformer}
       - LANGCHAIN_ENDPOINT=${LANGCHAIN_ENDPOINT-"https://api.smith.langchain.com"}

env.example

@@ -33,6 +33,14 @@ EMBEDDING_MODEL=sentence_transformer #or google-genai-embedding-001 openai, olla
 
 #OPENAI_API_KEY=sk-...
 
+#*****************************************************************
+# OrcaRouter
+#*****************************************************************
+# Only required when using an OrcaRouter LLM (LLM=orcarouter/<model>)
+
+#ORCAROUTER_BASE_URL=https://api.orcarouter.ai/v1
+#ORCAROUTER_API_KEY=sk-orca-...
+
 #*****************************************************************
 # AWS
 #*****************************************************************

pdf_bot.py

@@ -25,6 +25,8 @@
 username = os.getenv("NEO4J_USERNAME")
 password = os.getenv("NEO4J_PASSWORD")
 ollama_base_url = os.getenv("OLLAMA_BASE_URL")
+orcarouter_base_url = os.getenv("ORCAROUTER_BASE_URL")
+orcarouter_api_key = os.getenv("ORCAROUTER_API_KEY")
 embedding_model_name = os.getenv("EMBEDDING_MODEL")
 llm_name = os.getenv("LLM")
 # Remapping for Langchain Neo4j integration
@@ -48,7 +50,15 @@ def on_llm_new_token(self, token: str, **kwargs) -> None:
         self.container.markdown(self.text)
 
 
-llm = load_llm(llm_name, logger=logger, config={"ollama_base_url": ollama_base_url})
+llm = load_llm(
+    llm_name,
+    logger=logger,
+    config={
+        "ollama_base_url": ollama_base_url,
+        "orcarouter_base_url": orcarouter_base_url,
+        "orcarouter_api_key": orcarouter_api_key,
+    },
+)
 
 
 def main():

readme.md

@@ -11,10 +11,12 @@ Available variables:
 | Variable Name          | Default value                      | Description                                                             |
 |------------------------|------------------------------------|-------------------------------------------------------------------------|
 | OLLAMA_BASE_URL        | http://host.docker.internal:11434  | REQUIRED - URL to Ollama LLM API                                        |   
+| ORCAROUTER_BASE_URL    | https://api.orcarouter.ai/v1       | OPTIONAL - URL to the [OrcaRouter](https://www.orcarouter.ai) API       |
+| ORCAROUTER_API_KEY     |                                    | REQUIRED - Only if LLM=orcarouter/<model>                               |
 | NEO4J_URI              | neo4j://database:7687              | REQUIRED - URL to Neo4j database                                        |
 | NEO4J_USERNAME         | neo4j                              | REQUIRED - Username for Neo4j database                                  |
 | NEO4J_PASSWORD         | password                           | REQUIRED - Password for Neo4j database                                  |
-| LLM                    | llama2                             | REQUIRED - Can be any Ollama model tag, or gpt-4 or gpt-3.5 or claudev2 |
+| LLM                    | llama2                             | REQUIRED - Can be any Ollama model tag, or gpt-4 or gpt-3.5 or claudev2 or orcarouter/<model> |
 | EMBEDDING_MODEL        | sentence_transformer               | REQUIRED - Can be sentence_transformer, openai, aws, ollama or google-genai-embedding-001|
 | AWS_ACCESS_KEY_ID      |                                    | REQUIRED - Only if LLM=claudev2 or embedding_model=aws                  |
 | AWS_SECRET_ACCESS_KEY  |                                    | REQUIRED - Only if LLM=claudev2 or embedding_model=aws                  |
@@ -29,6 +31,7 @@ Available variables:
 ## LLM Configuration
 MacOS and Linux users can use any LLM that's available via Ollama. Check the "tags" section under the model page you want to use on https://ollama.ai/library and write the tag for the value of the environment variable `LLM=` in the `.env` file.
 All platforms can use GPT-3.5-turbo and GPT-4 (bring your own API keys for OpenAI models).
+All platforms can also use [OrcaRouter](https://www.orcarouter.ai) models by setting `LLM=orcarouter/<model>` (for example `orcarouter/fusion-mini`) and adding your `ORCAROUTER_API_KEY` to the `.env` file. OrcaRouter exposes an OpenAI-compatible endpoint, so the stack connects to it with the same LangChain `ChatOpenAI` integration used for OpenAI.
 
 **MacOS**
 Install [Ollama](https://ollama.ai) on MacOS and start it before running `docker compose up` using `ollama serve` in a separate terminal.