diff --git a/api.py b/api.py index d05b7d00..43fa6327 100644 --- a/api.py +++ b/api.py @@ -29,6 +29,8 @@ username = os.getenv("NEO4J_USERNAME") password = os.getenv("NEO4J_PASSWORD") ollama_base_url = os.getenv("OLLAMA_BASE_URL") +orcarouter_base_url = os.getenv("ORCAROUTER_BASE_URL") +orcarouter_api_key = os.getenv("ORCAROUTER_API_KEY") embedding_model_name = os.getenv("EMBEDDING_MODEL") llm_name = os.getenv("LLM") # Remapping for Langchain Neo4j integration @@ -47,7 +49,13 @@ create_vector_index(neo4j_graph) llm = load_llm( - llm_name, logger=BaseLogger(), config={"ollama_base_url": ollama_base_url} + llm_name, + logger=BaseLogger(), + config={ + "ollama_base_url": ollama_base_url, + "orcarouter_base_url": orcarouter_base_url, + "orcarouter_api_key": orcarouter_api_key, + }, ) llm_chain = configure_llm_only_chain(llm) diff --git a/bot.py b/bot.py index 4f3cf7fe..d5f7b8cc 100644 --- a/bot.py +++ b/bot.py @@ -22,6 +22,8 @@ username = os.getenv("NEO4J_USERNAME") password = os.getenv("NEO4J_PASSWORD") ollama_base_url = os.getenv("OLLAMA_BASE_URL") +orcarouter_base_url = os.getenv("ORCAROUTER_BASE_URL") +orcarouter_api_key = os.getenv("ORCAROUTER_API_KEY") embedding_model_name = os.getenv("EMBEDDING_MODEL") llm_name = os.getenv("LLM") # Remapping for Langchain Neo4j integration @@ -49,7 +51,15 @@ def on_llm_new_token(self, token: str, **kwargs) -> None: self.container.markdown(self.text) -llm = load_llm(llm_name, logger=logger, config={"ollama_base_url": ollama_base_url}) +llm = load_llm( + llm_name, + logger=logger, + config={ + "ollama_base_url": ollama_base_url, + "orcarouter_base_url": orcarouter_base_url, + "orcarouter_api_key": orcarouter_api_key, + }, +) llm_chain = configure_llm_only_chain(llm) rag_chain = configure_qa_rag_chain( diff --git a/chains.py b/chains.py index 58498822..b05abcc1 100644 --- a/chains.py +++ b/chains.py @@ -61,7 +61,19 @@ def load_embedding_model(embedding_model_name: str, logger=BaseLogger(), config= def load_llm(llm_name: str, logger=BaseLogger(), config={}): - if llm_name in ["gpt-4", "gpt-4o", "gpt-4-turbo"]: + if llm_name.startswith("orcarouter/"): + # OrcaRouter exposes its model namespace under the `orcarouter/` prefix on + # an OpenAI-compatible endpoint, so it can reuse ChatOpenAI directly. + logger.info(f"LLM: Using OrcaRouter: {llm_name}") + return ChatOpenAI( + temperature=0, + model_name=llm_name, + streaming=True, + base_url=config.get("orcarouter_base_url") + or "https://api.orcarouter.ai/v1", + api_key=config.get("orcarouter_api_key"), + ) + elif llm_name in ["gpt-4", "gpt-4o", "gpt-4-turbo"]: logger.info("LLM: Using GPT-4") return ChatOpenAI(temperature=0, model_name=llm_name, streaming=True) elif llm_name == "gpt-3.5": diff --git a/docker-compose.yml b/docker-compose.yml index 3a1bbc08..b26cd7aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,9 +102,11 @@ services: - NEO4J_URI=${NEO4J_URI-neo4j://database:7687} - NEO4J_PASSWORD=${NEO4J_PASSWORD-password} - NEO4J_USERNAME=${NEO4J_USERNAME-neo4j} - - OPENAI_API_KEY=${OPENAI_API_KEY-} + - OPENAI_API_KEY=${OPENAI_API_KEY-} - GOOGLE_API_KEY=${GOOGLE_API_KEY-} - OLLAMA_BASE_URL=${OLLAMA_BASE_URL-http://host.docker.internal:11434} + - ORCAROUTER_BASE_URL=${ORCAROUTER_BASE_URL-https://api.orcarouter.ai/v1} + - ORCAROUTER_API_KEY=${ORCAROUTER_API_KEY-} - LLM=${LLM-llama2} - EMBEDDING_MODEL=${EMBEDDING_MODEL-sentence_transformer} - LANGCHAIN_ENDPOINT=${LANGCHAIN_ENDPOINT-"https://api.smith.langchain.com"} @@ -144,6 +146,8 @@ services: - OPENAI_API_KEY=${OPENAI_API_KEY-} - GOOGLE_API_KEY=${GOOGLE_API_KEY-} - OLLAMA_BASE_URL=${OLLAMA_BASE_URL-http://host.docker.internal:11434} + - ORCAROUTER_BASE_URL=${ORCAROUTER_BASE_URL-https://api.orcarouter.ai/v1} + - ORCAROUTER_API_KEY=${ORCAROUTER_API_KEY-} - LLM=${LLM-llama2} - EMBEDDING_MODEL=${EMBEDDING_MODEL-sentence_transformer} - LANGCHAIN_ENDPOINT=${LANGCHAIN_ENDPOINT-"https://api.smith.langchain.com"} @@ -183,8 +187,10 @@ services: - NEO4J_PASSWORD=${NEO4J_PASSWORD-password} - NEO4J_USERNAME=${NEO4J_USERNAME-neo4j} - OPENAI_API_KEY=${OPENAI_API_KEY} - - GOOGLE_API_KEY=${GOOGLE_API_KEY} + - GOOGLE_API_KEY=${GOOGLE_API_KEY} - OLLAMA_BASE_URL=${OLLAMA_BASE_URL-http://host.docker.internal:11434} + - ORCAROUTER_BASE_URL=${ORCAROUTER_BASE_URL-https://api.orcarouter.ai/v1} + - ORCAROUTER_API_KEY=${ORCAROUTER_API_KEY-} - LLM=${LLM-llama2} - EMBEDDING_MODEL=${EMBEDDING_MODEL-sentence_transformer} - LANGCHAIN_ENDPOINT=${LANGCHAIN_ENDPOINT-"https://api.smith.langchain.com"} diff --git a/env.example b/env.example index 7d9574f3..e503f30b 100644 --- a/env.example +++ b/env.example @@ -33,6 +33,14 @@ EMBEDDING_MODEL=sentence_transformer #or google-genai-embedding-001 openai, olla #OPENAI_API_KEY=sk-... +#***************************************************************** +# OrcaRouter +#***************************************************************** +# Only required when using an OrcaRouter LLM (LLM=orcarouter/) + +#ORCAROUTER_BASE_URL=https://api.orcarouter.ai/v1 +#ORCAROUTER_API_KEY=sk-orca-... + #***************************************************************** # AWS #***************************************************************** diff --git a/pdf_bot.py b/pdf_bot.py index 96acee2c..d6a7d9b2 100644 --- a/pdf_bot.py +++ b/pdf_bot.py @@ -25,6 +25,8 @@ username = os.getenv("NEO4J_USERNAME") password = os.getenv("NEO4J_PASSWORD") ollama_base_url = os.getenv("OLLAMA_BASE_URL") +orcarouter_base_url = os.getenv("ORCAROUTER_BASE_URL") +orcarouter_api_key = os.getenv("ORCAROUTER_API_KEY") embedding_model_name = os.getenv("EMBEDDING_MODEL") llm_name = os.getenv("LLM") # Remapping for Langchain Neo4j integration @@ -48,7 +50,15 @@ def on_llm_new_token(self, token: str, **kwargs) -> None: self.container.markdown(self.text) -llm = load_llm(llm_name, logger=logger, config={"ollama_base_url": ollama_base_url}) +llm = load_llm( + llm_name, + logger=logger, + config={ + "ollama_base_url": ollama_base_url, + "orcarouter_base_url": orcarouter_base_url, + "orcarouter_api_key": orcarouter_api_key, + }, +) def main(): diff --git a/readme.md b/readme.md index 6ceb51e5..f9362ad5 100644 --- a/readme.md +++ b/readme.md @@ -11,10 +11,12 @@ Available variables: | Variable Name | Default value | Description | |------------------------|------------------------------------|-------------------------------------------------------------------------| | OLLAMA_BASE_URL | http://host.docker.internal:11434 | REQUIRED - URL to Ollama LLM API | +| ORCAROUTER_BASE_URL | https://api.orcarouter.ai/v1 | OPTIONAL - URL to the [OrcaRouter](https://www.orcarouter.ai) API | +| ORCAROUTER_API_KEY | | REQUIRED - Only if LLM=orcarouter/ | | NEO4J_URI | neo4j://database:7687 | REQUIRED - URL to Neo4j database | | NEO4J_USERNAME | neo4j | REQUIRED - Username for Neo4j database | | NEO4J_PASSWORD | password | REQUIRED - Password for Neo4j database | -| LLM | llama2 | REQUIRED - Can be any Ollama model tag, or gpt-4 or gpt-3.5 or claudev2 | +| LLM | llama2 | REQUIRED - Can be any Ollama model tag, or gpt-4 or gpt-3.5 or claudev2 or orcarouter/ | | EMBEDDING_MODEL | sentence_transformer | REQUIRED - Can be sentence_transformer, openai, aws, ollama or google-genai-embedding-001| | AWS_ACCESS_KEY_ID | | REQUIRED - Only if LLM=claudev2 or embedding_model=aws | | AWS_SECRET_ACCESS_KEY | | REQUIRED - Only if LLM=claudev2 or embedding_model=aws | @@ -29,6 +31,7 @@ Available variables: ## LLM Configuration MacOS and Linux users can use any LLM that's available via Ollama. Check the "tags" section under the model page you want to use on https://ollama.ai/library and write the tag for the value of the environment variable `LLM=` in the `.env` file. All platforms can use GPT-3.5-turbo and GPT-4 (bring your own API keys for OpenAI models). +All platforms can also use [OrcaRouter](https://www.orcarouter.ai) models by setting `LLM=orcarouter/` (for example `orcarouter/fusion-mini`) and adding your `ORCAROUTER_API_KEY` to the `.env` file. OrcaRouter exposes an OpenAI-compatible endpoint, so the stack connects to it with the same LangChain `ChatOpenAI` integration used for OpenAI. **MacOS** Install [Ollama](https://ollama.ai) on MacOS and start it before running `docker compose up` using `ollama serve` in a separate terminal.