Upload 1072 files

.circleci/config.yml

@@ -0,0 +1,2232 @@
+version: 2.1
+orbs:
+  codecov: codecov/[email protected]
+
+jobs:
+  local_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+
+      - run:
+          name: Show git commit hash
+          command: |
+            echo "Git commit hash: $CIRCLE_SHA1"
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r .circleci/requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+            pip install mypy
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
+            pip install langchain
+            pip install lunary==0.2.5
+            pip install "azure-identity==1.16.1"
+            pip install "langfuse==2.45.0"
+            pip install "logfire==0.29.0"
+            pip install numpydoc
+            pip install traceloop-sdk==0.21.1
+            pip install opentelemetry-api==1.25.0
+            pip install opentelemetry-sdk==1.25.0
+            pip install opentelemetry-exporter-otlp==1.25.0
+            pip install openai==1.54.0
+            pip install prisma==0.11.0
+            pip install "detect_secrets==1.5.0"
+            pip install "httpx==0.24.1"
+            pip install "respx==0.21.1"
+            pip install fastapi
+            pip install "gunicorn==21.2.0"
+            pip install "anyio==4.2.0"
+            pip install "aiodynamo==23.10.1"
+            pip install "asyncio==3.4.3"
+            pip install "apscheduler==3.10.4"
+            pip install "PyGithub==1.59.1"
+            pip install argon2-cffi
+            pip install "pytest-mock==3.12.0"
+            pip install python-multipart
+            pip install google-cloud-aiplatform
+            pip install prometheus-client==0.20.0
+            pip install "pydantic==2.7.1"
+            pip install "diskcache==5.6.1"
+            pip install "Pillow==10.3.0"
+            pip install "jsonschema==4.22.0"
+            pip install "pytest-xdist==3.6.1"
+            pip install "websockets==10.4"
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Run prisma ./docker/entrypoint.sh
+          command: |
+            set +e
+            chmod +x docker/entrypoint.sh
+            ./docker/entrypoint.sh
+            set -e
+      - run:
+          name: Black Formatting
+          command: |
+            cd litellm
+            python -m pip install black
+            python -m black .
+            cd ..
+      - run:
+          name: Linting Testing
+          command: |
+            cd litellm
+            python -m pip install types-requests types-setuptools types-redis types-PyYAML
+            if ! python -m mypy . --ignore-missing-imports; then
+              echo "mypy detected errors"
+              exit 1
+            fi
+            cd ..
+
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml local_testing_coverage.xml
+            mv .coverage local_testing_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - local_testing_coverage.xml
+            - local_testing_coverage
+  langfuse_logging_unit_tests:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+
+      - run:
+          name: Show git commit hash
+          command: |
+            echo "Git commit hash: $CIRCLE_SHA1"
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r .circleci/requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+            pip install mypy
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
+            pip install langchain
+            pip install lunary==0.2.5
+            pip install "azure-identity==1.16.1"
+            pip install "langfuse==2.45.0"
+            pip install "logfire==0.29.0"
+            pip install numpydoc
+            pip install traceloop-sdk==0.21.1
+            pip install opentelemetry-api==1.25.0
+            pip install opentelemetry-sdk==1.25.0
+            pip install opentelemetry-exporter-otlp==1.25.0
+            pip install openai==1.54.0
+            pip install prisma==0.11.0
+            pip install "detect_secrets==1.5.0"
+            pip install "httpx==0.24.1"
+            pip install "respx==0.21.1"
+            pip install fastapi
+            pip install "gunicorn==21.2.0"
+            pip install "anyio==4.2.0"
+            pip install "aiodynamo==23.10.1"
+            pip install "asyncio==3.4.3"
+            pip install "apscheduler==3.10.4"
+            pip install "PyGithub==1.59.1"
+            pip install argon2-cffi
+            pip install "pytest-mock==3.12.0"
+            pip install python-multipart
+            pip install google-cloud-aiplatform
+            pip install prometheus-client==0.20.0
+            pip install "pydantic==2.7.1"
+            pip install "diskcache==5.6.1"
+            pip install "Pillow==10.3.0"
+            pip install "jsonschema==4.22.0"
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Run prisma ./docker/entrypoint.sh
+          command: |
+            set +e
+            chmod +x docker/entrypoint.sh
+            ./docker/entrypoint.sh
+            set -e
+
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml langfuse_coverage.xml
+            mv .coverage langfuse_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - langfuse_coverage.xml
+            - langfuse_coverage
+  caching_unit_tests:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+
+      - run:
+          name: Show git commit hash
+          command: |
+            echo "Git commit hash: $CIRCLE_SHA1"
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r .circleci/requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+            pip install mypy
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
+            pip install langchain
+            pip install lunary==0.2.5
+            pip install "azure-identity==1.16.1"
+            pip install "langfuse==2.45.0"
+            pip install "logfire==0.29.0"
+            pip install numpydoc
+            pip install traceloop-sdk==0.21.1
+            pip install opentelemetry-api==1.25.0
+            pip install opentelemetry-sdk==1.25.0
+            pip install opentelemetry-exporter-otlp==1.25.0
+            pip install openai==1.54.0
+            pip install prisma==0.11.0
+            pip install "detect_secrets==1.5.0"
+            pip install "httpx==0.24.1"
+            pip install "respx==0.21.1"
+            pip install fastapi
+            pip install "gunicorn==21.2.0"
+            pip install "anyio==4.2.0"
+            pip install "aiodynamo==23.10.1"
+            pip install "asyncio==3.4.3"
+            pip install "apscheduler==3.10.4"
+            pip install "PyGithub==1.59.1"
+            pip install argon2-cffi
+            pip install "pytest-mock==3.12.0"
+            pip install python-multipart
+            pip install google-cloud-aiplatform
+            pip install prometheus-client==0.20.0
+            pip install "pydantic==2.7.1"
+            pip install "diskcache==5.6.1"
+            pip install "Pillow==10.3.0"
+            pip install "jsonschema==4.22.0"
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Run prisma ./docker/entrypoint.sh
+          command: |
+            set +e
+            chmod +x docker/entrypoint.sh
+            ./docker/entrypoint.sh
+            set -e
+
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "caching or cache"
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml caching_coverage.xml
+            mv .coverage caching_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - caching_coverage.xml
+            - caching_coverage
+  auth_ui_unit_tests:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Run prisma ./docker/entrypoint.sh
+          command: |
+            set +e
+            chmod +x docker/entrypoint.sh
+            ./docker/entrypoint.sh
+            set -e
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/proxy_admin_ui_tests -x --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml auth_ui_unit_tests_coverage.xml
+            mv .coverage auth_ui_unit_tests_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+
+      - persist_to_workspace:
+          root: .
+          paths:
+            - auth_ui_unit_tests_coverage.xml
+            - auth_ui_unit_tests_coverage
+  litellm_router_testing: # Runs all tests with the "router" keyword
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "respx==0.21.1"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest tests/local_testing tests/router_unit_tests --cov=litellm --cov-report=xml -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml litellm_router_coverage.xml
+            mv .coverage litellm_router_coverage
+      # Store test results
+      - store_test_results:
+          path: test-results
+
+      - persist_to_workspace:
+          root: .
+          paths:
+            - litellm_router_coverage.xml
+            - litellm_router_coverage
+  litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+    steps:
+      - checkout
+
+      - run:
+          name: Show git commit hash
+          command: |
+            echo "Git commit hash: $CIRCLE_SHA1"
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r .circleci/requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+            pip install mypy
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
+            pip install langchain
+            pip install lunary==0.2.5
+            pip install "azure-identity==1.16.1"
+            pip install "langfuse==2.45.0"
+            pip install "logfire==0.29.0"
+            pip install numpydoc
+            pip install traceloop-sdk==0.21.1
+            pip install opentelemetry-api==1.25.0
+            pip install opentelemetry-sdk==1.25.0
+            pip install opentelemetry-exporter-otlp==1.25.0
+            pip install openai==1.54.0
+            pip install prisma==0.11.0
+            pip install "detect_secrets==1.5.0"
+            pip install "httpx==0.24.1"
+            pip install "respx==0.21.1"
+            pip install fastapi
+            pip install "gunicorn==21.2.0"
+            pip install "anyio==4.2.0"
+            pip install "aiodynamo==23.10.1"
+            pip install "asyncio==3.4.3"
+            pip install "apscheduler==3.10.4"
+            pip install "PyGithub==1.59.1"
+            pip install argon2-cffi
+            pip install "pytest-mock==3.12.0"
+            pip install python-multipart
+            pip install google-cloud-aiplatform
+            pip install prometheus-client==0.20.0
+            pip install "pydantic==2.7.1"
+            pip install "diskcache==5.6.1"
+            pip install "Pillow==10.3.0"
+            pip install "jsonschema==4.22.0"
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+      - run:
+          name: Run prisma ./docker/entrypoint.sh
+          command: |
+            set +e
+            chmod +x docker/entrypoint.sh
+            ./docker/entrypoint.sh
+            set -e
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest tests/proxy_unit_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml litellm_proxy_unit_tests_coverage.xml
+            mv .coverage litellm_proxy_unit_tests_coverage
+      # Store test results
+      - store_test_results:
+          path: test-results
+
+      - persist_to_workspace:
+          root: .
+          paths:
+            - litellm_proxy_unit_tests_coverage.xml
+            - litellm_proxy_unit_tests_coverage
+  litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "respx==0.21.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest tests/local_testing/ -vv -k "assistants" --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml litellm_assistants_api_coverage.xml
+            mv .coverage litellm_assistants_api_coverage
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - litellm_assistants_api_coverage.xml
+            - litellm_assistants_api_coverage
+  load_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/load_tests -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+  llm_translation_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.21.1"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml llm_translation_coverage.xml
+            mv .coverage llm_translation_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - llm_translation_coverage.xml
+            - llm_translation_coverage
+  batches_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "respx==0.21.1"
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml batches_coverage.xml
+            mv .coverage batches_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - batches_coverage.xml
+            - batches_coverage
+  litellm_utils_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "respx==0.21.1"
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install numpydoc
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml litellm_utils_coverage.xml
+            mv .coverage litellm_utils_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - litellm_utils_coverage.xml
+            - litellm_utils_coverage
+
+  pass_through_unit_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.21.1"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml pass_through_unit_tests_coverage.xml
+            mv .coverage pass_through_unit_tests_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - pass_through_unit_tests_coverage.xml
+            - pass_through_unit_tests_coverage
+  image_gen_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.21.1"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml image_gen_coverage.xml
+            mv .coverage image_gen_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - image_gen_coverage.xml
+            - image_gen_coverage
+  logging_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install pytest-mock
+            pip install "respx==0.21.1"
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install "mlflow==2.17.2"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml logging_coverage.xml
+            mv .coverage logging_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - logging_coverage.xml
+            - logging_coverage
+  installing_litellm_on_python:
+    docker:
+      - image: circleci/python:3.8
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            pip install python-dotenv
+            pip install pytest
+            pip install tiktoken
+            pip install aiohttp
+            pip install openai
+            pip install click
+            pip install "boto3==1.34.34"
+            pip install jinja2
+            pip install tokenizers=="0.20.0"
+            pip install uvloop==0.21.0
+            pip install jsonschema
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/local_testing/test_basic_python_version.py
+
+  installing_litellm_on_python_3_13:
+    docker:
+      - image: cimg/python:3.13.1
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
+            pip install "tomli==2.2.1"
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/local_testing/test_basic_python_version.py
+  helm_chart_testing:
+    machine:
+      image: ubuntu-2204:2023.10.1  # Use machine executor instead of docker
+    resource_class: medium
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      # Install Helm
+      - run:
+          name: Install Helm
+          command: |
+            curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+      
+      # Install kind
+      - run:
+          name: Install Kind
+          command: |
+            curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+            chmod +x ./kind
+            sudo mv ./kind /usr/local/bin/kind
+      
+      # Install kubectl
+      - run:
+          name: Install kubectl
+          command: |
+            curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+            chmod +x kubectl
+            sudo mv kubectl /usr/local/bin/
+      
+      # Create kind cluster
+      - run:
+          name: Create Kind Cluster
+          command: |
+            kind create cluster --name litellm-test
+      
+      # Run helm lint
+      - run:
+          name: Run helm lint
+          command: |
+            helm lint ./deploy/charts/litellm-helm
+      
+      # Run helm tests
+      - run:
+          name: Run helm tests
+          command: |
+            helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml
+            # Wait for pod to be ready
+            echo "Waiting 30 seconds for pod to be ready..."
+            sleep 30
+            
+            # Print pod logs before running tests
+            echo "Printing pod logs..."
+            kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
+            
+            # Run the helm tests
+            helm test litellm --logs
+            helm test litellm --logs
+      
+      # Cleanup
+      - run:
+          name: Cleanup
+          command: |
+            kind delete cluster --name litellm-test
+          when: always  # This ensures cleanup runs even if previous steps fail
+
+
+  check_code_and_doc_quality:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project/litellm
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            pip install ruff
+            pip install pylint
+            pip install pyright
+            pip install beautifulsoup4
+            pip install .
+            curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+      - run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+      - run: ruff check ./litellm
+      # - run: python ./tests/documentation_tests/test_general_setting_keys.py
+      - run: python ./tests/code_coverage_tests/router_code_coverage.py
+      - run: python ./tests/code_coverage_tests/callback_manager_test.py
+      - run: python ./tests/code_coverage_tests/recursive_detector.py
+      - run: python ./tests/code_coverage_tests/test_router_strategy_async.py
+      - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
+      - run: python ./tests/code_coverage_tests/bedrock_pricing.py
+      - run: python ./tests/documentation_tests/test_env_keys.py
+      - run: python ./tests/documentation_tests/test_router_settings.py
+      - run: python ./tests/documentation_tests/test_api_docs.py
+      - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py
+      - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py
+      - run: python ./tests/documentation_tests/test_circular_imports.py
+      - run: helm lint ./deploy/charts/litellm-helm
+
+  db_migration_disable_update_check:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Build Docker image
+          command: |
+            docker build -t myapp . -f ./docker/Dockerfile.database
+      - run:
+          name: Run Docker container
+          command: |
+            docker run --name my-app \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e DISABLE_SCHEMA_UPDATE="True" \
+              -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
+              -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
+              -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
+              myapp:latest \
+              --config /app/config.yaml \
+              --port 4000 > docker_output.log 2>&1 || true
+      - run:
+          name: Display Docker logs
+          command: cat docker_output.log
+      - run:
+          name: Check for expected error
+          command: |
+            if grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two" docker_output.log; then
+              echo "Expected error found. Test passed."
+            else
+              echo "Expected error not found. Test failed."
+              cat docker_output.log
+              exit 1
+            fi
+
+  build_and_test:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Install Docker CLI (In case it's not already installed)
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+      - run:
+          name: Install Python 3.9
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.9 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            pip install "pytest==7.3.1"
+            pip install "pytest-asyncio==0.21.1"
+            pip install aiohttp
+            python -m pip install --upgrade pip
+            python -m pip install -r .circleci/requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install mypy
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
+            pip install langchain
+            pip install "langfuse>=2.0.0"
+            pip install "logfire==0.29.0"
+            pip install numpydoc
+            pip install prisma
+            pip install fastapi
+            pip install jsonschema
+            pip install "httpx==0.24.1"
+            pip install "gunicorn==21.2.0"
+            pip install "anyio==3.7.1"
+            pip install "aiodynamo==23.10.1"
+            pip install "asyncio==3.4.3"
+            pip install "PyGithub==1.59.1"
+            pip install "openai==1.54.0 "
+      - run:
+          name: Install Grype
+          command: |
+            curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
+      - run:
+          name: Build and Scan Docker Images
+          command: |
+            # Build and scan Dockerfile.database
+            echo "Building and scanning Dockerfile.database..."
+            docker build -t litellm-database:latest -f ./docker/Dockerfile.database .
+            grype litellm-database:latest --fail-on high
+
+            # Build and scan main Dockerfile
+            echo "Building and scanning main Dockerfile..."
+            docker build -t litellm:latest .
+            grype litellm:latest --fail-on high
+      - run:
+          name: Build Docker image
+          command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
+      - run:
+          name: Run Docker container
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e AZURE_API_KEY=$AZURE_API_KEY \
+              -e REDIS_HOST=$REDIS_HOST \
+              -e REDIS_PASSWORD=$REDIS_PASSWORD \
+              -e REDIS_PORT=$REDIS_PORT \
+              -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
+              -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
+              -e MISTRAL_API_KEY=$MISTRAL_API_KEY \
+              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+              -e GROQ_API_KEY=$GROQ_API_KEY \
+              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
+              -e COHERE_API_KEY=$COHERE_API_KEY \
+              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+              -e AWS_REGION_NAME=$AWS_REGION_NAME \
+              -e AUTO_INFER_REGION=True \
+              -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e USE_DDTRACE=True \
+              -e DD_API_KEY=$DD_API_KEY \
+              -e DD_SITE=$DD_SITE \
+              -e LITELLM_LICENSE=$LITELLM_LICENSE \
+              -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
+              -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
+              -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
+              -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
+              --name my-app \
+              -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug \
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start outputting logs
+          command: docker logs -f my-app
+          background: true
+      - run:
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
+          no_output_timeout: 120m
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+  e2e_openai_misc_endpoints:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Install Docker CLI (In case it's not already installed)
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+      - run:
+          name: Install Python 3.9
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.9 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            pip install "pytest==7.3.1"
+            pip install "pytest-asyncio==0.21.1"
+            pip install aiohttp
+            python -m pip install --upgrade pip
+            python -m pip install -r .circleci/requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install mypy
+            pip install "jsonlines==4.0.0"
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
+            pip install langchain
+            pip install "langfuse>=2.0.0"
+            pip install "logfire==0.29.0"
+            pip install numpydoc
+            pip install prisma
+            pip install fastapi
+            pip install jsonschema
+            pip install "httpx==0.24.1"
+            pip install "gunicorn==21.2.0"
+            pip install "anyio==3.7.1"
+            pip install "aiodynamo==23.10.1"
+            pip install "asyncio==3.4.3"
+            pip install "PyGithub==1.59.1"
+            pip install "openai==1.54.0 "
+            # Run pytest and generate JUnit XML report
+      - run:
+          name: Build Docker image
+          command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
+      - run:
+          name: Run Docker container
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e AZURE_API_KEY=$AZURE_BATCHES_API_KEY \
+              -e AZURE_API_BASE=$AZURE_BATCHES_API_BASE \
+              -e AZURE_API_VERSION="2024-05-01-preview" \
+              -e REDIS_HOST=$REDIS_HOST \
+              -e REDIS_PASSWORD=$REDIS_PASSWORD \
+              -e REDIS_PORT=$REDIS_PORT \
+              -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
+              -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
+              -e MISTRAL_API_KEY=$MISTRAL_API_KEY \
+              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+              -e GROQ_API_KEY=$GROQ_API_KEY \
+              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
+              -e COHERE_API_KEY=$COHERE_API_KEY \
+              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+              -e AWS_REGION_NAME=$AWS_REGION_NAME \
+              -e AUTO_INFER_REGION=True \
+              -e USE_DDTRACE=True \
+              -e DD_API_KEY=$DD_API_KEY \
+              -e DD_SITE=$DD_SITE \
+              -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e LITELLM_LICENSE=$LITELLM_LICENSE \
+              -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
+              -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
+              -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
+              -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
+              --name my-app \
+              -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug \
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start outputting logs
+          command: docker logs -f my-app
+          background: true
+      - run:
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -s -vv tests/openai_misc_endpoints_tests --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+  proxy_logging_guardrails_model_info_tests:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Install Docker CLI (In case it's not already installed)
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+      - run:
+          name: Install Python 3.9
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.9 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            pip install "pytest==7.3.1"
+            pip install "pytest-asyncio==0.21.1"
+            pip install aiohttp
+            python -m pip install --upgrade pip
+            python -m pip install -r .circleci/requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install mypy
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
+            pip install langchain
+            pip install "langfuse>=2.0.0"
+            pip install "logfire==0.29.0"
+            pip install numpydoc
+            pip install prisma
+            pip install fastapi
+            pip install jsonschema
+            pip install "httpx==0.24.1"
+            pip install "gunicorn==21.2.0"
+            pip install "anyio==3.7.1"
+            pip install "aiodynamo==23.10.1"
+            pip install "asyncio==3.4.3"
+            pip install "PyGithub==1.59.1"
+            pip install "openai==1.54.0 "
+      - run:
+          name: Build Docker image
+          command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
+      - run:
+          name: Run Docker container
+          # intentionally give bad redis credentials here
+          # the OTEL test - should get this as a trace
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e REDIS_HOST=$REDIS_HOST \
+              -e REDIS_PASSWORD=$REDIS_PASSWORD \
+              -e REDIS_PORT=$REDIS_PORT \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e LITELLM_LICENSE=$LITELLM_LICENSE \
+              -e OTEL_EXPORTER="in_memory" \
+              -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \
+              -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \
+              -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
+              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+              -e USE_DDTRACE=True \
+              -e DD_API_KEY=$DD_API_KEY \
+              -e DD_SITE=$DD_SITE \
+              -e AWS_REGION_NAME=$AWS_REGION_NAME \
+              -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
+              -e COHERE_API_KEY=$COHERE_API_KEY \
+              -e GCS_FLUSH_INTERVAL="1" \
+              --name my-app \
+              -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
+              -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug \
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start outputting logs
+          command: docker logs -f my-app
+          background: true
+      - run:
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout:
+            120m
+            # Clean up first container
+      - run:
+          name: Stop and remove first container
+          command: |
+            docker stop my-app
+            docker rm my-app
+
+      # Second Docker Container Run with Different Config
+      # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license
+      - run:
+          name: Run Second Docker container
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e REDIS_HOST=$REDIS_HOST \
+              -e REDIS_PASSWORD=$REDIS_PASSWORD \
+              -e REDIS_PORT=$REDIS_PORT \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e LITELLM_LICENSE="bad-license" \
+              --name my-app-3 \
+              -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug
+
+      - run:
+          name: Start outputting logs for second container
+          command: docker logs -f my-app-2
+          background: true
+
+      - run:
+          name: Wait for second app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+
+      - run:
+          name: Run second round of tests
+          command: |
+            python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+          no_output_timeout: 120m
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+
+  proxy_store_model_in_db_tests:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Install Docker CLI (In case it's not already installed)
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+      - run:
+          name: Install Python 3.9
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.9 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            pip install "pytest==7.3.1"
+            pip install "pytest-asyncio==0.21.1"
+            pip install aiohttp
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest-asyncio==0.21.1"
+      - run:
+          name: Build Docker image
+          command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
+      - run:
+          name: Run Docker container
+          # intentionally give bad redis credentials here
+          # the OTEL test - should get this as a trace
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e STORE_MODEL_IN_DB="True" \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e LITELLM_LICENSE=$LITELLM_LICENSE \
+              --name my-app \
+              -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug \
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start outputting logs
+          command: docker logs -f my-app
+          background: true
+      - run:
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout:
+            120m
+            # Clean up first container
+      
+  proxy_build_from_pip_tests:
+    # Change from docker to machine executor
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      # Remove Docker CLI installation since it's already available in machine executor
+      - run:
+          name: Install Python 3.13
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.13 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            pip install "pytest==7.3.1"
+            pip install "pytest-asyncio==0.21.1"
+            pip install aiohttp
+            python -m pip install --upgrade pip
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install mypy
+      - run:
+          name: Build Docker image
+          command: |
+            cd docker/build_from_pip
+            docker build -t my-app:latest -f Dockerfile.build_from_pip .
+      - run:
+          name: Run Docker container
+          # intentionally give bad redis credentials here
+          # the OTEL test - should get this as a trace
+          command: |
+            cd docker/build_from_pip
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e REDIS_HOST=$REDIS_HOST \
+              -e REDIS_PASSWORD=$REDIS_PASSWORD \
+              -e REDIS_PORT=$REDIS_PORT \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e LITELLM_LICENSE=$LITELLM_LICENSE \
+              -e OTEL_EXPORTER="in_memory" \
+              -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \
+              -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \
+              -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
+              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+              -e AWS_REGION_NAME=$AWS_REGION_NAME \
+              -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
+              -e COHERE_API_KEY=$COHERE_API_KEY \
+              -e USE_DDTRACE=True \
+              -e DD_API_KEY=$DD_API_KEY \
+              -e DD_SITE=$DD_SITE \
+              -e GCS_FLUSH_INTERVAL="1" \
+              --name my-app \
+              -v $(pwd)/litellm_config.yaml:/app/config.yaml \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug \
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start outputting logs
+          command: docker logs -f my-app
+          background: true
+      - run:
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Run tests
+          command: |
+            python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+          no_output_timeout:
+            120m
+            # Clean up first container
+      - run:
+          name: Stop and remove first container
+          command: |
+            docker stop my-app
+            docker rm my-app
+  proxy_pass_through_endpoint_tests:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Install Docker CLI (In case it's not already installed)
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+      - run:
+          name: Install Python 3.9
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.9 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install aiohttp
+            pip install "openai==1.54.0 "
+            pip install "assemblyai==0.37.0"
+            python -m pip install --upgrade pip
+            pip install "pydantic==2.7.1"
+            pip install "pytest==7.3.1"
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "boto3==1.34.34"
+            pip install mypy
+            pip install pyarrow
+            pip install numpydoc
+            pip install prisma
+            pip install fastapi
+            pip install jsonschema
+            pip install "httpx==0.24.1"
+            pip install "anyio==3.7.1"
+            pip install "asyncio==3.4.3"
+            pip install "PyGithub==1.59.1"
+            pip install "google-cloud-aiplatform==1.59.0"
+            pip install anthropic
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Build Docker image
+          command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
+      - run:
+          name: Run Docker container
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e GEMINI_API_KEY=$GEMINI_API_KEY \
+              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
+              -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
+              -e USE_DDTRACE=True \
+              -e DD_API_KEY=$DD_API_KEY \
+              -e DD_SITE=$DD_SITE \
+              -e LITELLM_LICENSE=$LITELLM_LICENSE \
+              --name my-app \
+              -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
+              -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug \
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start outputting logs
+          command: docker logs -f my-app
+          background: true
+      - run:
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      # New steps to run Node.js test
+      - run:
+          name: Install Node.js
+          command: |
+            curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash -
+            sudo apt-get install -y nodejs
+            node --version
+            npm --version
+
+      - run:
+          name: Install Node.js dependencies
+          command: |
+            npm install @google-cloud/vertexai
+            npm install @google/generative-ai
+            npm install --save-dev jest
+
+      - run:
+          name: Run Vertex AI, Google AI Studio Node.js tests
+          command: |
+            npx jest tests/pass_through_tests --verbose
+          no_output_timeout: 30m
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      # Store test results
+      - store_test_results:
+          path: test-results
+
+  upload-coverage:
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      # Check file locations
+      - run:
+          name: Check coverage file location
+          command: |
+            echo "Current directory:"
+            ls -la
+            echo "\nContents of tests/llm_translation:"
+            ls -la tests/llm_translation
+      - run:
+          name: Combine Coverage
+          command: |
+            python -m venv venv
+            . venv/bin/activate
+            pip install coverage
+            coverage combine llm_translation_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage
+            coverage xml
+      - codecov/upload:
+          file: ./coverage.xml
+
+  publish_to_pypi:
+    docker:
+      - image: cimg/python:3.8
+    working_directory: ~/project
+
+    environment:
+      TWINE_USERNAME: __token__
+
+    steps:
+      - checkout
+
+      - run:
+          name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup
+          command: |
+            cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
+
+      - run:
+          name: Check if litellm dir, tests dir, or pyproject.toml was modified
+          command: |
+            if [ -n "$(git diff --name-only $CIRCLE_SHA1^..$CIRCLE_SHA1 | grep -E 'pyproject\.toml|litellm/|tests/')" ]; then
+              echo "litellm, tests, or pyproject.toml updated"
+            else
+              echo "No changes to litellm, tests, or pyproject.toml. Skipping PyPI publish."
+              circleci step halt
+            fi
+
+      - run:
+          name: Checkout code
+          command: git checkout $CIRCLE_SHA1
+
+      # Check if setup.py is modified and publish to PyPI
+      - run:
+          name: PyPI publish
+          command: |
+            echo "Install TOML package."
+            python -m pip install toml
+            VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
+            PACKAGE_NAME=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['name'])")
+            if ! pip show -v $PACKAGE_NAME | grep -q "Version: ${VERSION}"; then
+                echo "pyproject.toml modified"
+                echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
+                python -m pip install --upgrade pip
+                pip install build
+                pip install wheel
+                pip install --upgrade twine setuptools
+                rm -rf build dist
+
+                echo "Building package"
+                python -m build
+
+                echo "Twine upload to dist"
+                echo "Contents of dist directory:"
+                ls dist/
+                twine upload --verbose dist/*
+            else
+                echo "Version ${VERSION} of package is already published on PyPI. Skipping PyPI publish."
+                circleci step halt
+            fi
+      - run:
+          name: Trigger Github Action for new Docker Container + Trigger Stable Release Testing
+          command: |
+            echo "Install TOML package."
+            python3 -m pip install toml
+            VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
+            echo "LiteLLM Version ${VERSION}"
+            curl -X POST \
+              -H "Accept: application/vnd.github.v3+json" \
+              -H "Authorization: Bearer $GITHUB_TOKEN" \
+              "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
+              -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
+            echo "triggering stable release server for version ${VERSION} and commit ${CIRCLE_SHA1}"
+            curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}"
+
+  e2e_ui_testing:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Install Docker CLI (In case it's not already installed)
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+      - run:
+          name: Install Python 3.9
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.9 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            npm install -D @playwright/test
+            npm install @google-cloud/vertexai
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-asyncio==0.21.1"
+            pip install aiohttp
+            pip install "openai==1.54.0 "
+            python -m pip install --upgrade pip
+            pip install "pydantic==2.7.1"
+            pip install "pytest==7.3.1"
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install mypy
+            pip install pyarrow
+            pip install numpydoc
+            pip install prisma
+            pip install fastapi
+            pip install jsonschema
+            pip install "httpx==0.24.1"
+            pip install "anyio==3.7.1"
+            pip install "asyncio==3.4.3"
+      - run:
+          name: Install Playwright Browsers
+          command: |
+            npx playwright install
+      - run:
+          name: Build Docker image
+          command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
+      - run:
+          name: Run Docker container
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$SMALL_DATABASE_URL \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e UI_USERNAME="admin" \
+              -e UI_PASSWORD="gm" \
+              -e LITELLM_LICENSE=$LITELLM_LICENSE \
+              --name my-app \
+              -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start outputting logs
+          command: docker logs -f my-app
+          background: true
+      - run:
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Run Playwright Tests
+          command: |
+            npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
+          no_output_timeout: 120m
+      - store_test_results:
+          path: test-results
+
+  test_nonroot_image:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Build Docker image
+          command: |
+            docker build -t non_root_image:latest . -f ./docker/Dockerfile.non_root
+      - run:
+          name: Install Container Structure Test
+          command: |
+            curl -LO https://github.com/GoogleContainerTools/container-structure-test/releases/download/v1.19.3/container-structure-test-linux-amd64
+            chmod +x container-structure-test-linux-amd64
+            sudo mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test
+      - run:
+          name: Run Container Structure Test
+          command: |
+            container-structure-test test --image non_root_image:latest --config docker/tests/nonroot.yaml
+
+  test_bad_database_url:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Build Docker image
+          command: |
+            docker build -t myapp . -f ./docker/Dockerfile.non_root
+      - run:
+          name: Run Docker container with bad DATABASE_URL
+          command: |
+            docker run --name my-app \
+              -p 4000:4000 \
+              -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
+              myapp:latest \
+              --port 4000 > docker_output.log 2>&1 || true
+      - run:
+          name: Display Docker logs
+          command: cat docker_output.log
+      - run:
+          name: Check for expected error
+          command: |
+            if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
+               grep -q "httpx.ConnectError: All connection attempts failed" docker_output.log && \
+               grep -q "ERROR:    Application startup failed. Exiting." docker_output.log; then
+              echo "Expected error found. Test passed."
+            else
+              echo "Expected error not found. Test failed."
+              cat docker_output.log
+              exit 1
+            fi
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - local_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - langfuse_logging_unit_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - caching_unit_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - litellm_proxy_unit_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - litellm_assistants_api_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - litellm_router_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - check_code_and_doc_quality:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - auth_ui_unit_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - e2e_ui_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - build_and_test:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - e2e_openai_misc_endpoints:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - proxy_logging_guardrails_model_info_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - proxy_store_model_in_db_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - proxy_build_from_pip_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - proxy_pass_through_endpoint_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - llm_translation_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - batches_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - litellm_utils_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - pass_through_unit_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - image_gen_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - logging_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - upload-coverage:
+          requires:
+            - llm_translation_testing
+            - batches_testing
+            - litellm_utils_testing
+            - pass_through_unit_testing
+            - image_gen_testing
+            - logging_testing
+            - litellm_router_testing
+            - caching_unit_tests
+            - litellm_proxy_unit_testing
+            - langfuse_logging_unit_tests
+            - local_testing
+            - litellm_assistants_api_testing
+            - auth_ui_unit_tests
+      - db_migration_disable_update_check:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - installing_litellm_on_python:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - installing_litellm_on_python_3_13:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - helm_chart_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - load_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - test_bad_database_url:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - publish_to_pypi:
+          requires:
+            - local_testing
+            - build_and_test
+            - e2e_openai_misc_endpoints
+            - load_testing
+            - test_bad_database_url
+            - llm_translation_testing
+            - batches_testing
+            - litellm_utils_testing
+            - pass_through_unit_testing
+            - image_gen_testing
+            - logging_testing
+            - litellm_router_testing
+            - caching_unit_tests
+            - langfuse_logging_unit_tests
+            - litellm_assistants_api_testing
+            - auth_ui_unit_tests
+            - db_migration_disable_update_check
+            - e2e_ui_testing
+            - litellm_proxy_unit_testing
+            - installing_litellm_on_python
+            - installing_litellm_on_python_3_13
+            - proxy_logging_guardrails_model_info_tests
+            - proxy_store_model_in_db_tests
+            - proxy_build_from_pip_tests
+            - proxy_pass_through_endpoint_tests
+            - check_code_and_doc_quality
+          filters:
+            branches:
+              only:
+                - main

.circleci/requirements.txt

@@ -0,0 +1,13 @@
+# used by CI/CD testing
+openai==1.54.0 
+python-dotenv
+tiktoken
+importlib_metadata
+cohere
+redis
+anthropic
+orjson==3.9.15
+pydantic==2.7.1
+google-cloud-aiplatform==1.43.0
+fastapi-sso==0.10.0
+uvloop==0.21.0

.devcontainer/devcontainer.json

@@ -0,0 +1,52 @@
+{
+	"name": "Python 3.11",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:3.11-bookworm",
+	// https://github.com/devcontainers/images/tree/main/src/python
+	// https://mcr.microsoft.com/en-us/product/devcontainers/python/tags
+
+	// "build": {
+	// 	"dockerfile": "Dockerfile",
+	// 	"context": ".."
+	// },
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			"settings": {},
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"GitHub.copilot",
+				"GitHub.copilot-chat",
+				"ms-python.autopep8"
+			]
+		}
+	},
+	
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [4000],
+
+	"containerEnv": {
+		"LITELLM_LOG": "DEBUG"
+	},
+
+	// Use 'portsAttributes' to set default properties for specific forwarded ports. 
+	// More info: https://containers.dev/implementors/json_reference/#port-attributes
+	"portsAttributes": {
+		"4000": {
+			"label": "LiteLLM Server",
+			"onAutoForward": "notify"
+		}
+	},
+
+	// More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "litellm",
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
+}

.dockerignore

@@ -0,0 +1,12 @@
+docs
+cookbook
+.circleci
+.github
+tests
+.git
+.github
+.circleci
+.devcontainer
+*.tgz
+log.txt
+docker/Dockerfile.*

.env.example

@@ -0,0 +1,22 @@
+# OpenAI
+OPENAI_API_KEY = ""
+OPENAI_API_BASE = ""
+# Cohere
+COHERE_API_KEY = ""
+# OpenRouter
+OR_SITE_URL = ""
+OR_APP_NAME = "LiteLLM Example app"
+OR_API_KEY = ""
+# Azure API base URL
+AZURE_API_BASE = ""
+# Azure API version
+AZURE_API_VERSION = ""
+# Azure API key
+AZURE_API_KEY = ""
+# Replicate
+REPLICATE_API_KEY = ""
+REPLICATE_API_TOKEN = ""
+# Anthropic
+ANTHROPIC_API_KEY = ""
+# Infisical
+INFISICAL_TOKEN = ""

.flake8

@@ -0,0 +1,46 @@
+[flake8]
+ignore =
+    # The following ignores can be removed when formatting using black
+    W191,W291,W292,W293,W391,W504
+    E101,E111,E114,E116,E117,E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,
+    E201,E202,E221,E222,E225,E226,E231,E241,E251,E252,E261,E265,E271,E272,E275,
+    E301,E302,E303,E305,E306,
+    # line break before binary operator
+    W503,
+    # inline comment should start with '# '
+    E262,
+    # too many leading '#' for block comment
+    E266,
+    # multiple imports on one line
+    E401,
+    # module level import not at top of file
+    E402,
+    # Line too long (82 > 79 characters)
+    E501,
+    # comparison to None should be 'if cond is None:'
+    E711,
+    # comparison to True should be 'if cond is True:' or 'if cond:'
+    E712,
+    # do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`
+    E721,
+    # do not use bare 'except'
+    E722,
+    # x is imported but unused
+    F401,
+    # 'from . import *' used; unable to detect undefined names
+    F403,
+    # x may be undefined, or defined from star imports:
+    F405,
+    # f-string is missing placeholders
+    F541,
+    # dictionary key '' repeated with different values
+    F601,
+    # redefinition of unused x from line 123
+    F811,
+    # undefined name x
+    F821,
+    # local variable x is assigned to but never used
+    F841,
+
+# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
+extend-ignore = E203

.git-blame-ignore-revs

@@ -0,0 +1,10 @@
+# Add the commit hash of any commit you want to ignore in `git blame` here.
+# One commit hash per line.
+#
+# The GitHub Blame UI will use this file automatically!
+#
+# Run this command to always ignore formatting commits in `git blame`
+#     git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# Update pydantic code to fix warnings (GH-3600)
+876840e9957bc7e9f7d6a2b58c4d7c53dad16481

.gitattributes

@@ -1,35 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ipynb linguist-vendored
+*.png filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+docs/my-website/img/*.png filter=lfs diff=lfs merge=lfs -text
+docs/my-website/img/*.gif filter=lfs diff=lfs merge=lfs -text

.github/FUNDING.yml

@@ -0,0 +1,13 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+custom: https://buy.stripe.com/9AQ03Kd3P91o0Q8bIS

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,49 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+      placeholder: Tell us what you see!
+      value: "A bug happened!"
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+  - type: dropdown
+    id: ml-ops-team
+    attributes:
+      label: Are you a ML Ops Team?
+      description: This helps us prioritize your requests correctly 
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: What LiteLLM version are you on ? 
+      placeholder: v1.53.1
+    validations:
+      required: true
+  - type: input
+    id: contact
+    attributes:
+      label: Twitter / LinkedIn details 
+      description: We announce new features on Twitter + LinkedIn. If this issue leads to an announcement, and you'd like a mention, we'll gladly shout you out!
+      placeholder: ex. @krrish_dh / https://www.linkedin.com/in/krish-d/
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Schedule Demo
+    url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
+    about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
+  - name: Discord
+    url: https://discord.com/invite/wuPM9dRgDw
+    about: Join 250+ LiteLLM community members! 

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,42 @@
+name: 🚀 Feature Request 
+description: Submit a proposal/request for a new LiteLLM feature.
+title: "[Feature]: "
+labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for making LiteLLM better! 
+  - type: textarea
+    id: the-feature
+    attributes:
+      label: The Feature
+      description: A clear and concise description of the feature proposal
+      placeholder: Tell us what you want!
+    validations:
+      required: true
+  - type: textarea
+    id: motivation
+    attributes:
+      label: Motivation, pitch
+      description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
+    validations:
+      required: true
+  - type: dropdown
+    id: ml-ops-team
+    attributes:
+      label: Are you a ML Ops Team?
+      description: This helps us prioritize your requests correctly 
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true
+  - type: input
+    id: contact
+    attributes:
+      label: Twitter / LinkedIn details 
+      description: We announce new features on Twitter + LinkedIn. When this is announced, and you'd like a mention, we'll gladly shout you out!
+      placeholder: ex. @krrish_dh / https://www.linkedin.com/in/krish-d/
+    validations:
+      required: false

.github/actions/helm-oci-chart-releaser/action.yml

@@ -0,0 +1,77 @@
+name: Helm OCI Chart Releaser
+description: Push Helm charts to OCI-based (Docker) registries
+author: sergeyshaykhullin
+branding:
+  color: yellow
+  icon: upload-cloud
+inputs:
+  name:
+    required: true
+    description: Chart name
+  repository:
+    required: true
+    description: Chart repository name
+  tag:
+    required: true
+    description: Chart version
+  app_version:
+    required: true
+    description: App version
+  path:
+    required: false
+    description: Chart path (Default 'charts/{name}')
+  registry:
+    required: true
+    description: OCI registry
+  registry_username:
+    required: true
+    description: OCI registry username
+  registry_password:
+    required: true
+    description: OCI registry password
+  update_dependencies:
+    required: false
+    default: 'false'
+    description: Update chart dependencies before packaging (Default 'false')
+outputs:
+  image:
+    value: ${{ steps.output.outputs.image }}
+    description: Chart image (Default '{registry}/{repository}/{image}:{tag}')
+runs:
+  using: composite
+  steps:
+    - name: Helm | Login
+      shell: bash
+      run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
+      env:
+        HELM_EXPERIMENTAL_OCI: '1'
+    
+    - name: Helm | Dependency
+      if: inputs.update_dependencies == 'true'
+      shell: bash
+      run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
+      env:
+        HELM_EXPERIMENTAL_OCI: '1'
+
+    - name: Helm | Package
+      shell: bash
+      run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
+      env:
+        HELM_EXPERIMENTAL_OCI: '1'
+
+    - name: Helm | Push
+      shell: bash
+      run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
+      env:
+        HELM_EXPERIMENTAL_OCI: '1'
+
+    - name: Helm | Logout
+      shell: bash
+      run: helm registry logout ${{ inputs.registry }}
+      env:
+        HELM_EXPERIMENTAL_OCI: '1'
+
+    - name: Helm | Output
+      id: output
+      shell: bash
+      run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT

.github/dependabot.yaml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    groups:
+      github-actions:
+        patterns:
+          - "*"

.github/pull_request_template.md

@@ -0,0 +1,29 @@
+## Title
+
+<!-- e.g. "Implement user authentication feature" -->
+
+## Relevant issues
+
+<!-- e.g. "Fixes #000" -->
+
+## Type
+
+<!-- Select the type of Pull Request -->
+<!-- Keep only the necessary ones -->
+
+🆕 New Feature
+🐛 Bug Fix
+🧹 Refactoring
+📖 Documentation
+🚄 Infrastructure
+✅ Test
+
+## Changes
+
+<!-- List of changes -->
+
+## [REQUIRED] Testing - Attach a screenshot of any new tests passing locally
+If UI changes, send a screenshot/GIF of working UI fixes
+
+<!-- Test procedure -->
+

.github/template.yaml

@@ -0,0 +1,94 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: >
+  llmlite-service
+
+  SAM Template for llmlite-service
+
+# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
+Globals:
+  Function:
+    Timeout: 600
+    MemorySize: 128
+    Environment:
+      Variables:
+        WORKER_CONFIG: !Ref WorkerConfigParameter
+
+Parameters:
+  AliasParameter:
+    Type: String
+    Default: live
+  WorkerConfigParameter:
+    Type: String
+    Description: Sample environment variable
+    Default: '{"model": null, "alias": null, "api_base": null, "api_version": "2023-07-01-preview", "debug": false, "temperature": null, "max_tokens": null, "request_timeout": 600, "max_budget": null, "telemetry": true, "drop_params": false, "add_function_to_prompt": false, "headers": null, "save": false, "config": null, "use_queue": false}'
+
+Resources:
+  MyUrlFunctionPermissions:
+    Type: AWS::Lambda::Permission
+    Properties:
+      FunctionName: !Ref URL
+      Action: lambda:InvokeFunctionUrl
+      Principal: "*"
+      FunctionUrlAuthType: NONE
+
+  Function:
+    Type: AWS::Serverless::Function
+    Properties:
+      FunctionName: !Sub "${AWS::StackName}-function"
+      CodeUri: "./litellm"
+      Handler: proxy/lambda.handler
+      Runtime: python3.11
+      AutoPublishAlias: !Ref AliasParameter
+      Architectures:
+       - x86_64
+      DeploymentPreference:
+        Type: AllAtOnce
+        Alarms:
+          - !Ref NewVersionErrorMetricGreaterThanZeroAlarm
+
+  NewVersionErrorMetricGreaterThanZeroAlarm:
+    Type: "AWS::CloudWatch::Alarm"
+    Properties:
+      AlarmDescription: Lambda Function Error > 0
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions:
+        - Name: Resource
+          Value: !Sub "${Function}:live"
+        - Name: FunctionName
+          Value: !Ref Function
+        - Name: ExecutedVersion
+          Value: !GetAtt Function.Version.Version
+      EvaluationPeriods: 1
+      Unit: Count
+      MetricName: Errors
+      Namespace: AWS/Lambda
+      Period: 60
+      Statistic: Sum
+      Threshold: 0
+
+  URL:
+    Type: AWS::Lambda::Url
+    DependsOn: FunctionAliaslive
+    Properties:
+      AuthType: NONE
+      Qualifier: live
+      TargetFunctionArn: !GetAtt Function.Arn
+
+Outputs:
+  FunctionARN:
+    Description: "Lambda Function ARN"
+    Value: !GetAtt Function.Arn
+
+  FunctionUrl:
+    Description: "Lambda Function URL Endpoint"
+    Value:
+      Fn::GetAtt: URL.FunctionUrl
+
+  FunctionVersion:
+    Description: "Lambda Function Version"
+    Value: !GetAtt Function.Version.Version
+  
+  FunctionNewAlarmARN:
+    Description: "Lambda Function New Alarm ARN"
+    Value: !GetAtt NewVersionErrorMetricGreaterThanZeroAlarm.Arn

.github/workflows/auto_update_price_and_context_window.yml

@@ -0,0 +1,28 @@
+name: Updates model_prices_and_context_window.json and Create Pull Request
+
+on:
+  schedule:
+    - cron: "0 0 * * 0"  # Run every Sundays at midnight
+    #- cron: "0 0 * * *" # Run daily at midnight
+
+jobs:
+  auto_update_price_and_context_window:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Dependencies
+        run: |
+          pip install aiohttp
+      - name: Update JSON Data
+        run: |
+          python ".github/workflows/auto_update_price_and_context_window_file.py"
+      - name: Create Pull Request
+        run: |
+          git add model_prices_and_context_window.json 
+          git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" 
+          gh pr create --title "Update model_prices_and_context_window.json file" \
+            --body "Automated update for model_prices_and_context_window.json" \
+            --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
+            --base main
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}

.github/workflows/auto_update_price_and_context_window_file.py

@@ -0,0 +1,121 @@
+import asyncio
+import aiohttp
+import json
+
+# Asynchronously fetch data from a given URL
+async def fetch_data(url):
+    try:
+        # Create an asynchronous session
+        async with aiohttp.ClientSession() as session:
+            # Send a GET request to the URL
+            async with session.get(url) as resp:
+                # Raise an error if the response status is not OK
+                resp.raise_for_status()
+                # Parse the response JSON
+                resp_json = await resp.json()
+                print("Fetch the data from URL.")
+                # Return the 'data' field from the JSON response
+                return resp_json['data']
+    except Exception as e:
+        # Print an error message if fetching data fails
+        print("Error fetching data from URL:", e)
+        return None
+
+# Synchronize local data with remote data
+def sync_local_data_with_remote(local_data, remote_data):
+    # Update existing keys in local_data with values from remote_data
+    for key in (set(local_data) & set(remote_data)):
+        local_data[key].update(remote_data[key])
+
+    # Add new keys from remote_data to local_data
+    for key in (set(remote_data) - set(local_data)):
+        local_data[key] = remote_data[key]
+
+# Write data to the json file
+def write_to_file(file_path, data):
+    try:
+        # Open the file in write mode
+        with open(file_path, "w") as file:
+            # Dump the data as JSON into the file
+            json.dump(data, file, indent=4)
+        print("Values updated successfully.")
+    except Exception as e:
+        # Print an error message if writing to file fails
+        print("Error updating JSON file:", e)
+
+# Update the existing models and add the missing models
+def transform_remote_data(data):
+    transformed = {}
+    for row in data:
+        # Add the fields 'max_tokens' and 'input_cost_per_token'
+        obj = {
+            "max_tokens": row["context_length"],
+            "input_cost_per_token": float(row["pricing"]["prompt"]),
+        }
+
+        # Add 'max_output_tokens' as a field if it is not None
+        if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None:
+            obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"])
+
+        # Add the field 'output_cost_per_token'
+        obj.update({
+            "output_cost_per_token": float(row["pricing"]["completion"]),
+        })
+
+        # Add field 'input_cost_per_image' if it exists and is non-zero
+        if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0:
+            obj['input_cost_per_image'] = float(row["pricing"]["image"])
+
+        # Add the fields 'litellm_provider' and 'mode'
+        obj.update({
+            "litellm_provider": "openrouter",
+            "mode": "chat"
+        })
+
+        # Add the 'supports_vision' field if the modality is 'multimodal'
+        if row.get('architecture', {}).get('modality') == 'multimodal':
+            obj['supports_vision'] = True
+
+        # Use a composite key to store the transformed object
+        transformed[f'openrouter/{row["id"]}'] = obj
+
+    return transformed
+
+
+# Load local data from a specified file
+def load_local_data(file_path):
+    try:
+        # Open the file in read mode
+        with open(file_path, "r") as file:
+            # Load and return the JSON data
+            return json.load(file)
+    except FileNotFoundError:
+        # Print an error message if the file is not found
+        print("File not found:", file_path)
+        return None
+    except json.JSONDecodeError as e:
+        # Print an error message if JSON decoding fails
+        print("Error decoding JSON:", e)
+        return None
+
+def main():
+    local_file_path = "model_prices_and_context_window.json"  # Path to the local data file
+    url = "https://openrouter.ai/api/v1/models"  # URL to fetch remote data
+
+    # Load local data from file
+    local_data = load_local_data(local_file_path)
+    # Fetch remote data asynchronously
+    remote_data = asyncio.run(fetch_data(url))
+    # Transform the fetched remote data
+    remote_data = transform_remote_data(remote_data)
+
+    # If both local and remote data are available, synchronize and save
+    if local_data and remote_data:
+        sync_local_data_with_remote(local_data, remote_data)
+        write_to_file(local_file_path, local_data)
+    else:
+        print("Failed to fetch model data from either local file or URL.")
+
+# Entry point of the script
+if __name__ == "__main__":
+    main()

.github/workflows/ghcr_deploy.yml

@@ -0,0 +1,374 @@
+# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
+name: Build, Publish LiteLLM Docker Image. New Release
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "The tag version you want to build"
+      release_type:
+        description: "The release type you want to build. Can be 'latest', 'stable', 'dev'"
+        type: string
+        default: "latest"
+      commit_hash:
+        description: "Commit hash"
+        required: true
+
+# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  CHART_NAME: litellm-helm
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  # print commit hash, tag, and release type
+  print:
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          echo "Commit hash: ${{ github.event.inputs.commit_hash }}"
+          echo "Tag: ${{ github.event.inputs.tag }}"
+          echo "Release type: ${{ github.event.inputs.release_type }}"
+  docker-hub-deploy:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.commit_hash }}
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} 
+      -
+        name: Build and push litellm-database image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          file: ./docker/Dockerfile.database
+          tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }}
+      -
+        name: Build and push litellm-spend-logs image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          file: ./litellm-js/spend-logs/Dockerfile
+          tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
+      
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      #
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.commit_hash }}
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # Configure multi platform Docker builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
+          
+  build-and-push-image-database:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.commit_hash }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for database Dockerfile
+        id: meta-database
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database
+      # Configure multi platform Docker builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
+
+      - name: Build and push Database Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: ./docker/Dockerfile.database
+          push: true
+          tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} 
+          labels: ${{ steps.meta-database.outputs.labels }} 
+          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
+            
+  build-and-push-image-non_root:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.commit_hash }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for non_root Dockerfile
+        id: meta-non_root
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root
+      # Configure multi platform Docker builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
+
+      - name: Build and push non_root Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: ./docker/Dockerfile.non_root
+          push: true
+          tags: ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }} 
+          labels: ${{ steps.meta-non_root.outputs.labels }} 
+          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
+  
+  build-and-push-image-spend-logs:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.commit_hash }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for spend-logs Dockerfile
+        id: meta-spend-logs
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs
+      # Configure multi platform Docker builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
+
+      - name: Build and push Database Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: ./litellm-js/spend-logs/Dockerfile
+          push: true
+          tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
+          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
+
+  build-and-push-helm-chart:
+    if: github.event.inputs.release_type  != 'dev'
+    needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: lowercase github.repository_owner
+        run: |
+          echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
+    
+      - name: Get LiteLLM Latest Tag
+        id: current_app_tag
+        shell: bash
+        run: |
+          LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
+          if [ -z "${LATEST_TAG}" ]; then
+            echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
+          else
+            echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
+          fi
+
+      - name: Get last published chart version
+        id: current_version
+        shell: bash
+        run: |
+          CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
+          if [ -z "${CHART_LIST}" ]; then
+            echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
+          else
+            printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
+          fi
+        env:
+          HELM_EXPERIMENTAL_OCI: '1'
+
+      # Automatically update the helm chart version one "patch" level
+      - name: Bump release version
+        id: bump_version
+        uses: christian-draeger/[email protected]
+        with:
+          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
+          version-fragment: 'bug'
+
+      - uses: ./.github/actions/helm-oci-chart-releaser
+        with:
+          name: ${{ env.CHART_NAME }}
+          repository: ${{ env.REPO_OWNER }}
+          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
+          app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
+          path: deploy/charts/${{ env.CHART_NAME }}
+          registry: ${{ env.REGISTRY }}
+          registry_username: ${{ github.actor }}
+          registry_password: ${{ secrets.GITHUB_TOKEN }}
+          update_dependencies: true
+
+  release:
+    name: "New LiteLLM Release"
+    needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
+
+    runs-on: "ubuntu-latest"
+   
+    steps:
+      - name: Display version
+        run: echo "Current version is ${{ github.event.inputs.tag }}"
+      - name: "Set Release Tag"
+        run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
+      - name: Display release tag
+        run: echo "RELEASE_TAG is $RELEASE_TAG"
+      - name: "Create release"
+        uses: "actions/github-script@v6"
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          script: |
+            const commitHash = "${{ github.event.inputs.commit_hash}}";
+            console.log("Commit Hash:", commitHash); // Add this line for debugging
+            try {
+              const response = await github.rest.repos.createRelease({
+                draft: false,
+                generate_release_notes: true,
+                target_commitish: commitHash,
+                name: process.env.RELEASE_TAG,
+                owner: context.repo.owner,
+                prerelease: false,
+                repo: context.repo.repo,
+                tag_name: process.env.RELEASE_TAG,
+              });
+      
+              core.exportVariable('RELEASE_ID', response.data.id);
+              core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url);
+            } catch (error) {
+              core.setFailed(error.message);
+            }
+      - name: Fetch Release Notes
+        id: release-notes
+        uses: actions/github-script@v6
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          script: |
+            try {
+              const response = await github.rest.repos.getRelease({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                release_id: process.env.RELEASE_ID,
+              });
+              const formattedBody = JSON.stringify(response.data.body).slice(1, -1);
+              return formattedBody;
+            } catch (error) {
+              core.setFailed(error.message);
+            }
+        env:
+          RELEASE_ID: ${{ env.RELEASE_ID }}
+      - name: Github Releases To Discord
+        env:
+          WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
+          REALEASE_TAG: ${{ env.RELEASE_TAG }}
+          RELEASE_NOTES: ${{ steps.release-notes.outputs.result }}
+        run: |
+          curl -H "Content-Type: application/json" -X POST -d '{
+            "content": "New LiteLLM release '"${RELEASE_TAG}"'",
+            "username": "Release Changelog",
+            "avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png",
+            "embeds": [
+              {
+                "title": "Changelog for LiteLLM '"${RELEASE_TAG}"'",
+                "description": "'"${RELEASE_NOTES}"'",
+                "color": 2105893
+              }
+            ]
+          }' $WEBHOOK_URL
+

.github/workflows/ghcr_helm_deploy.yml

@@ -0,0 +1,67 @@
+# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
+name: Build, Publish LiteLLM Helm Chart. New Release
+on:
+  workflow_dispatch:
+    inputs:
+      chartVersion:
+        description: "Update the helm chart's version to this"
+
+# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  REPO_OWNER: ${{github.repository_owner}}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:        
+  build-and-push-helm-chart:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      
+      - name: lowercase github.repository_owner
+        run: |
+          echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
+
+      - name: Get LiteLLM Latest Tag
+        id: current_app_tag
+        uses: WyriHaximus/[email protected]
+
+      - name: Get last published chart version
+        id: current_version
+        shell: bash
+        run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
+        env:
+          HELM_EXPERIMENTAL_OCI: '1'
+
+      # Automatically update the helm chart version one "patch" level
+      - name: Bump release version
+        id: bump_version
+        uses: christian-draeger/[email protected]
+        with:
+          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
+          version-fragment: 'bug'
+
+      - name: Lint helm chart
+        run: helm lint deploy/charts/litellm-helm
+
+      - uses: ./.github/actions/helm-oci-chart-releaser
+        with:
+          name: litellm-helm
+          repository: ${{ env.REPO_OWNER }}
+          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
+          app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
+          path: deploy/charts/litellm-helm
+          registry: ${{ env.REGISTRY }}
+          registry_username: ${{ github.actor }}
+          registry_password: ${{ secrets.GITHUB_TOKEN }}
+          update_dependencies: true
+  

.github/workflows/interpret_load_test.py

@@ -0,0 +1,113 @@
+import csv
+import os
+from github import Github
+
+
+def interpret_results(csv_file):
+    with open(csv_file, newline="") as csvfile:
+        csvreader = csv.DictReader(csvfile)
+        rows = list(csvreader)
+        """
+        in this csv reader
+        - Create 1 new column "Status"
+        - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅"
+        - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌"
+        - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns
+        """
+
+        # Add a new column "Status"
+        for row in rows:
+            median_response_time = float(
+                row["Median Response Time"].strip().rstrip("ms")
+            )
+            average_response_time = float(
+                row["Average Response Time"].strip().rstrip("s")
+            )
+
+            request_count = int(row["Request Count"])
+            failure_count = int(row["Failure Count"])
+
+            failure_percent = round((failure_count / request_count) * 100, 2)
+
+            # Determine status based on conditions
+            if (
+                median_response_time < 300
+                and average_response_time < 300
+                and failure_percent < 5
+            ):
+                row["Status"] = "Passed ✅"
+            else:
+                row["Status"] = "Failed ❌"
+
+        # Construct Markdown table header
+        markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |"
+        markdown_table += (
+            "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
+        )
+
+        # Construct Markdown table rows
+        for row in rows:
+            markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |"
+    print("markdown table: ", markdown_table)
+    return markdown_table
+
+
+if __name__ == "__main__":
+    csv_file = "load_test_stats.csv"  # Change this to the path of your CSV file
+    markdown_table = interpret_results(csv_file)
+
+    # Update release body with interpreted results
+    github_token = os.getenv("GITHUB_TOKEN")
+    g = Github(github_token)
+    repo = g.get_repo(
+        "BerriAI/litellm"
+    )  # Replace with your repository's username and name
+    latest_release = repo.get_latest_release()
+    print("got latest release: ", latest_release)
+    print(latest_release.title)
+    print(latest_release.tag_name)
+
+    release_version = latest_release.title
+
+    print("latest release body: ", latest_release.body)
+    print("markdown table: ", markdown_table)
+
+    # check if "Load Test LiteLLM Proxy Results" exists
+    existing_release_body = latest_release.body
+    if "Load Test LiteLLM Proxy Results" in latest_release.body:
+        # find the "Load Test LiteLLM Proxy Results" section and delete it
+        start_index = latest_release.body.find("Load Test LiteLLM Proxy Results")
+        existing_release_body = latest_release.body[:start_index]
+
+    docker_run_command = f"""
+\n\n
+## Docker Run LiteLLM Proxy
+
+```
+docker run \\
+-e STORE_MODEL_IN_DB=True \\
+-p 4000:4000 \\
+ghcr.io/berriai/litellm:main-{release_version}
+```
+    """
+    print("docker run command: ", docker_run_command)
+
+    new_release_body = (
+        existing_release_body
+        + docker_run_command
+        + "\n\n"
+        + "### Don't want to maintain your internal proxy? get in touch 🎉"
+        + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+        + "\n\n"
+        + "## Load Test LiteLLM Proxy Results"
+        + "\n\n"
+        + markdown_table
+    )
+    print("new release body: ", new_release_body)
+    try:
+        latest_release.update_release(
+            name=latest_release.tag_name,
+            message=new_release_body,
+        )
+    except Exception as e:
+        print(e)

.github/workflows/label-mlops.yml

@@ -0,0 +1,17 @@
+name: Label ML Ops Team Issues
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-mlops-label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if ML Ops Team is selected
+        uses: actions-ecosystem/action-add-labels@v1
+        if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes')
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          labels: "mlops user request"

.github/workflows/load_test.yml

@@ -0,0 +1,59 @@
+name: Test Locust Load Test
+
+on:
+  workflow_run:
+    workflows: ["Build, Publish LiteLLM Docker Image. New Release"]
+    types:
+      - completed
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install PyGithub
+      - name: re-deploy proxy
+        run: |
+          echo "Current working directory: $PWD"
+          ls
+          python ".github/workflows/redeploy_proxy.py"
+        env:
+          LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }}
+          LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }}
+        working-directory: ${{ github.workspace }}
+      - name: Run Load Test
+        id: locust_run
+        uses: BerriAI/locust-github-action@master
+        with:
+          LOCUSTFILE: ".github/workflows/locustfile.py"
+          URL:  "https://post-release-load-test-proxy.onrender.com/"
+          USERS: "20"
+          RATE: "20"
+          RUNTIME: "300s"
+      - name: Process Load Test Stats
+        run: |
+          echo "Current working directory: $PWD"
+          ls
+          python ".github/workflows/interpret_load_test.py"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        working-directory: ${{ github.workspace }}
+      - name: Upload CSV as Asset to Latest Release
+        uses: xresloader/upload-to-github-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          file: "load_test_stats.csv;load_test.html"
+          update_latest_release: true
+          tag_name: "load-test"
+          overwrite: true

.github/workflows/locustfile.py

@@ -0,0 +1,28 @@
+from locust import HttpUser, task, between
+
+
+class MyUser(HttpUser):
+    wait_time = between(1, 5)
+
+    @task
+    def chat_completion(self):
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": "Bearer sk-ZoHqrLIs2-5PzJrqBaviAA",
+            # Include any additional headers you may need for authentication, etc.
+        }
+
+        # Customize the payload with "model" and "messages" keys
+        payload = {
+            "model": "fake-openai-endpoint",
+            "messages": [
+                {"role": "system", "content": "You are a chat bot."},
+                {"role": "user", "content": "Hello, how are you?"},
+            ],
+            # Add more data as necessary
+        }
+
+        # Make a POST request to the "chat/completions" endpoint
+        response = self.client.post("chat/completions", json=payload, headers=headers)
+
+        # Print or log the response if needed

.github/workflows/main.yml

@@ -0,0 +1,34 @@
+name: Publish Dev Release to PyPI
+
+on:
+  workflow_dispatch:
+  
+jobs:
+  publish-dev-release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8  # Adjust the Python version as needed
+
+      - name: Install dependencies
+        run: pip install toml twine
+
+      - name: Read version from pyproject.toml
+        id: read-version
+        run: |
+          version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
+          printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
+
+      - name: Check if version exists on PyPI
+        id: check-version
+        run: |
+          set -e
+          if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then
+            echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish."
+     

.github/workflows/read_pyproject_version.yml

@@ -0,0 +1,31 @@
+name: Read Version from pyproject.toml
+
+on:
+  push:
+    branches:
+      - main  # Change this to the default branch of your repository
+
+jobs:
+  read-version:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8  # Adjust the Python version as needed
+
+      - name: Install dependencies
+        run: pip install toml
+
+      - name: Read version from pyproject.toml
+        id: read-version
+        run: |
+          version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
+          printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
+
+      - name: Display version
+        run: echo "Current version is $LITELLM_VERSION"

.github/workflows/redeploy_proxy.py

@@ -0,0 +1,20 @@
+"""
+
+redeploy_proxy.py
+"""
+
+import os
+import requests
+import time
+
+# send a get request to this endpoint
+deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1")
+response = requests.get(deploy_hook1, timeout=20)
+
+
+deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2")
+response = requests.get(deploy_hook2, timeout=20)
+
+print("SENT GET REQUESTS to re-deploy proxy")
+print("sleeeping.... for 60s")
+time.sleep(60)

.github/workflows/reset_stable.yml

@@ -0,0 +1,39 @@
+name: Reset litellm_stable branch
+
+on:
+  release:
+    types: [published, created]
+jobs:
+  update-stable-branch:
+    if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Reset litellm_stable_release_branch branch to the release commit
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          
+          # Fetch all branches and tags
+          git fetch --all
+
+          # Check if the litellm_stable_release_branch branch exists
+          if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then
+            echo "litellm_stable_release_branch branch exists."
+            git checkout litellm_stable_release_branch
+          else
+            echo "litellm_stable_release_branch branch does not exist. Creating it."
+            git checkout -b litellm_stable_release_branch
+          fi
+
+          # Reset litellm_stable_release_branch branch to the release commit
+          git reset --hard $GITHUB_SHA
+
+          # Push the updated litellm_stable_release_branch branch
+          git push origin litellm_stable_release_branch --force

.github/workflows/results_stats.csv

@@ -0,0 +1,27 @@
+Date,"Ben 
+Ashley",Tom Brooks,Jimmy Cooney,"Sue 
+Daniels",Berlinda Fong,Terry Jones,Angelina Little,Linda Smith
+10/1,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE
+10/2,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/3,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/4,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/5,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/6,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/7,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/8,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/9,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/10,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/11,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/12,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/13,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/14,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/15,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/16,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/17,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/18,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/19,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/20,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/21,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/22,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+10/23,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
+Total,0,1,1,1,1,1,0,1

.github/workflows/stale.yml

@@ -0,0 +1,20 @@
+name: "Stale Issue Management"
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # Runs daily at midnight UTC
+  workflow_dispatch:
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v8
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
+          stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
+          days-before-stale: 90        # Revert to 60 days
+          days-before-close: 7         # Revert to 7 days
+          stale-issue-label: "stale"
+          operations-per-run: 1000

.github/workflows/update_release.py

@@ -0,0 +1,54 @@
+import os
+import requests
+from datetime import datetime
+
+# GitHub API endpoints
+GITHUB_API_URL = "https://api.github.com"
+REPO_OWNER = "BerriAI"
+REPO_NAME = "litellm"
+
+# GitHub personal access token (required for uploading release assets)
+GITHUB_ACCESS_TOKEN = os.environ.get("GITHUB_ACCESS_TOKEN")
+
+# Headers for GitHub API requests
+headers = {
+    "Accept": "application/vnd.github+json",
+    "Authorization": f"Bearer {GITHUB_ACCESS_TOKEN}",
+    "X-GitHub-Api-Version": "2022-11-28",
+}
+
+# Get the latest release
+releases_url = f"{GITHUB_API_URL}/repos/{REPO_OWNER}/{REPO_NAME}/releases/latest"
+response = requests.get(releases_url, headers=headers)
+latest_release = response.json()
+print("Latest release:", latest_release)
+
+# Upload an asset to the latest release
+upload_url = latest_release["upload_url"].split("{?")[0]
+asset_name = "results_stats.csv"
+asset_path = os.path.join(os.getcwd(), asset_name)
+print("upload_url:", upload_url)
+
+with open(asset_path, "rb") as asset_file:
+    asset_data = asset_file.read()
+
+upload_payload = {
+    "name": asset_name,
+    "label": "Load test results",
+    "created_at": datetime.utcnow().isoformat() + "Z",
+}
+
+upload_headers = headers.copy()
+upload_headers["Content-Type"] = "application/octet-stream"
+
+upload_response = requests.post(
+    upload_url,
+    headers=upload_headers,
+    data=asset_data,
+    params=upload_payload,
+)
+
+if upload_response.status_code == 201:
+    print(f"Asset '{asset_name}' uploaded successfully to the latest release.")
+else:
+    print(f"Failed to upload asset. Response: {upload_response.text}")

.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+repos:
+-   repo: local
+    hooks:
+    -   id: pyright
+        name: pyright
+        entry: pyright
+        language: system
+        types: [python]
+        files: ^litellm/
+    -   id: isort
+        name: isort
+        entry: isort
+        language: system
+        types: [python]
+        files: litellm/.*\.py
+        exclude: ^litellm/__init__.py$
+-   repo: https://github.com/psf/black
+    rev: 24.2.0
+    hooks:
+    - id: black
+-   repo: https://github.com/pycqa/flake8
+    rev: 7.0.0  # The version of flake8 to use
+    hooks:
+    -  id: flake8
+       exclude: ^litellm/tests/|^litellm/proxy/tests/
+       additional_dependencies: [flake8-print]
+       files: litellm/.*\.py
+    # -  id: flake8
+    #    name: flake8 (router.py function length)
+    #    files: ^litellm/router\.py$
+    #    args: [--max-function-length=40]
+    # #    additional_dependencies: [flake8-functions]
+-   repo: https://github.com/python-poetry/poetry
+    rev: 1.8.0
+    hooks:
+      - id: poetry-check
+-   repo: local
+    hooks:
+    -   id: check-files-match
+        name: Check if files match
+        entry: python3 ci_cd/check_files_match.py
+        language: system
+    # -   id: check-file-length
+    #     name: Check file length
+    #     entry: python check_file_length.py
+    #     args: ["10000"]  # set your desired maximum number of lines
+    #     language: python
+    #     files: litellm/.*\.py
+    #     exclude: ^litellm/tests/

Dockerfile

@@ -0,0 +1,97 @@
+# Base image for building
+ARG LITELLM_BUILD_IMAGE=python:3.13.1-slim
+
+# Runtime image
+ARG LITELLM_RUNTIME_IMAGE=python:3.13.1-slim
+# Builder stage
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Set the shell to bash
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Install build dependencies
+RUN apt-get clean && apt-get update && \
+    apt-get install -y gcc python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir build
+
+# Copy the current directory contents into the container at /app
+COPY . .
+
+# Build Admin UI
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+# Build the package
+RUN rm -rf dist/* && python -m build
+
+# There should be only one wheel file now, assume the build only creates one
+RUN ls -1 dist/*.whl | head -1
+
+# Install the package
+RUN pip install dist/*.whl
+
+# install dependencies as wheels
+RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+
+# Runtime stage
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+# Update dependencies and clean up - handles debian security issue
+RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+# Copy the current directory contents into the container at /app
+COPY . .
+RUN ls -la /app
+
+# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
+COPY --from=builder /app/dist/*.whl .
+COPY --from=builder /wheels/ /wheels/
+
+# Install the built wheel using pip; again using a wildcard if it's the only file
+RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+
+# install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0
+# ensure pyjwt is used, not jwt
+RUN pip install redisvl==0.0.7 --no-deps --no-cache-dir && \
+    pip uninstall jwt -y && \
+    pip uninstall PyJWT -y && \
+    pip install PyJWT==2.9.0 --no-cache-dir
+
+# Build Admin UI
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+### Prisma Handling for Non-Root #################################################
+# Prisma allows you to specify the binary cache directory to use
+ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
+
+RUN pip install --no-cache-dir nodejs-bin prisma
+
+# Make a /non-existent folder and assign chown to nobody
+RUN mkdir -p /nonexistent && \
+    chown -R nobody:nogroup /app && \
+    chown -R nobody:nogroup /nonexistent && \
+    chown -R nobody:nogroup /usr/local/lib/python3.13/site-packages/prisma/
+
+RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
+
+# Run Prisma generate as user = nobody
+USER nobody
+
+RUN prisma generate
+### End of Prisma Handling for Non-Root #########################################
+
+EXPOSE 4000/tcp
+
+# # Set your entrypoint and command
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
+
+# Append "--detailed_debug" to the end of CMD to view detailed debug logs
+# CMD ["--port", "4000", "--detailed_debug"]
+CMD ["--port", "4000"]

LICENSE

@@ -0,0 +1,26 @@
+Portions of this software are licensed as follows:
+
+* All content that resides under the "enterprise/" directory of this repository, if that directory exists, is licensed under the license defined in "enterprise/LICENSE".
+* Content outside of the above mentioned directories or restrictions above is available under the MIT license as defined below.
+---
+MIT License
+
+Copyright (c) 2023 Berri AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,11 +1,8 @@
 ---
-title: TestLLM
-emoji: 🏆
-colorFrom: indigo
-colorTo: purple
+title: litellm
+emoji: 🐳
+colorFrom: purple
+colorTo: gray
 sdk: docker
-pinned: false
-license: mit
+app_port: 4000
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

ci_cd/check_file_length.py

@@ -0,0 +1,28 @@
+import sys
+
+
+def check_file_length(max_lines, filenames):
+    bad_files = []
+    for filename in filenames:
+        with open(filename, "r") as file:
+            lines = file.readlines()
+            if len(lines) > max_lines:
+                bad_files.append((filename, len(lines)))
+    return bad_files
+
+
+if __name__ == "__main__":
+    max_lines = int(sys.argv[1])
+    filenames = sys.argv[2:]
+
+    bad_files = check_file_length(max_lines, filenames)
+    if bad_files:
+        bad_files.sort(
+            key=lambda x: x[1], reverse=True
+        )  # Sort files by length in descending order
+        for filename, length in bad_files:
+            print(f"{filename}: {length} lines")
+
+        sys.exit(1)
+    else:
+        sys.exit(0)

ci_cd/check_files_match.py

@@ -0,0 +1,32 @@
+import sys
+import filecmp
+import shutil
+
+
+def main(argv=None):
+    print(
+        "Comparing model_prices_and_context_window and litellm/model_prices_and_context_window_backup.json files... checking if they match."
+    )
+
+    file1 = "model_prices_and_context_window.json"
+    file2 = "litellm/model_prices_and_context_window_backup.json"
+
+    cmp_result = filecmp.cmp(file1, file2, shallow=False)
+
+    if cmp_result:
+        print(f"Passed! Files {file1} and {file2} match.")
+        return 0
+    else:
+        print(
+            f"Failed! Files {file1} and {file2} do not match. Copying content from {file1} to {file2}."
+        )
+        copy_content(file1, file2)
+        return 1
+
+
+def copy_content(source, destination):
+    shutil.copy2(source, destination)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

codecov.yaml

@@ -0,0 +1,32 @@
+component_management:
+  individual_components:
+    - component_id: "Router"
+      paths:
+        - "router"
+    - component_id: "LLMs"
+      paths:
+        - "*/llms/*"
+    - component_id: "Caching"
+      paths:
+        - "*/caching/*"
+        - ".*redis.*"
+    - component_id: "litellm_logging"
+      paths:
+        - "*/integrations/*"
+        - ".*litellm_logging.*"
+    - component_id: "Proxy_Authentication"
+      paths:
+        - "*/proxy/auth/**"
+comment:
+  layout: "header, diff, flags, components"  # show component info in the PR comment
+
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1% # at maximum allow project coverage to drop by 1%
+    patch:
+      default:
+        target: auto
+        threshold: 0% # patch coverage should be 100%

db_scripts/create_views.py

@@ -0,0 +1,209 @@
+"""
+python script to pre-create all views required by LiteLLM Proxy Server
+"""
+
+import asyncio
+
+# Enter your DATABASE_URL here
+
+from prisma import Prisma
+
+db = Prisma(
+    http={
+        "timeout": 60000,
+    },
+)
+
+
+async def check_view_exists():  # noqa: PLR0915
+    """
+    Checks if the LiteLLM_VerificationTokenView and MonthlyGlobalSpend exists in the user's db.
+
+    LiteLLM_VerificationTokenView: This view is used for getting the token + team data in user_api_key_auth
+
+    MonthlyGlobalSpend: This view is used for the admin view to see global spend for this month
+
+    If the view doesn't exist, one will be created.
+    """
+
+    # connect to dB
+    await db.connect()
+    try:
+        # Try to select one row from the view
+        await db.query_raw("""SELECT 1 FROM "LiteLLM_VerificationTokenView" LIMIT 1""")
+        print("LiteLLM_VerificationTokenView Exists!")  # noqa
+    except Exception:
+        # If an error occurs, the view does not exist, so create it
+        await db.execute_raw(
+            """
+                CREATE VIEW "LiteLLM_VerificationTokenView" AS
+                SELECT 
+                v.*, 
+                t.spend AS team_spend, 
+                t.max_budget AS team_max_budget, 
+                t.tpm_limit AS team_tpm_limit, 
+                t.rpm_limit AS team_rpm_limit
+                FROM "LiteLLM_VerificationToken" v
+                LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id;
+            """
+        )
+
+        print("LiteLLM_VerificationTokenView Created!")  # noqa
+
+    try:
+        await db.query_raw("""SELECT 1 FROM "MonthlyGlobalSpend" LIMIT 1""")
+        print("MonthlyGlobalSpend Exists!")  # noqa
+    except Exception:
+        sql_query = """
+        CREATE OR REPLACE VIEW "MonthlyGlobalSpend" AS 
+        SELECT
+        DATE("startTime") AS date, 
+        SUM("spend") AS spend 
+        FROM 
+        "LiteLLM_SpendLogs" 
+        WHERE 
+        "startTime" >= (CURRENT_DATE - INTERVAL '30 days')
+        GROUP BY 
+        DATE("startTime");
+        """
+        await db.execute_raw(query=sql_query)
+
+        print("MonthlyGlobalSpend Created!")  # noqa
+
+    try:
+        await db.query_raw("""SELECT 1 FROM "Last30dKeysBySpend" LIMIT 1""")
+        print("Last30dKeysBySpend Exists!")  # noqa
+    except Exception:
+        sql_query = """
+        CREATE OR REPLACE VIEW "Last30dKeysBySpend" AS
+        SELECT 
+        L."api_key", 
+        V."key_alias",
+        V."key_name",
+        SUM(L."spend") AS total_spend
+        FROM
+        "LiteLLM_SpendLogs" L
+        LEFT JOIN 
+        "LiteLLM_VerificationToken" V
+        ON
+        L."api_key" = V."token"
+        WHERE
+        L."startTime" >= (CURRENT_DATE - INTERVAL '30 days')
+        GROUP BY
+        L."api_key", V."key_alias", V."key_name"
+        ORDER BY
+        total_spend DESC;
+        """
+        await db.execute_raw(query=sql_query)
+
+        print("Last30dKeysBySpend Created!")  # noqa
+
+    try:
+        await db.query_raw("""SELECT 1 FROM "Last30dModelsBySpend" LIMIT 1""")
+        print("Last30dModelsBySpend Exists!")  # noqa
+    except Exception:
+        sql_query = """
+        CREATE OR REPLACE VIEW "Last30dModelsBySpend" AS
+        SELECT
+        "model",
+        SUM("spend") AS total_spend
+        FROM
+        "LiteLLM_SpendLogs"
+        WHERE
+        "startTime" >= (CURRENT_DATE - INTERVAL '30 days')
+        AND "model" != ''
+        GROUP BY
+        "model"
+        ORDER BY
+        total_spend DESC;
+        """
+        await db.execute_raw(query=sql_query)
+
+        print("Last30dModelsBySpend Created!")  # noqa
+    try:
+        await db.query_raw("""SELECT 1 FROM "MonthlyGlobalSpendPerKey" LIMIT 1""")
+        print("MonthlyGlobalSpendPerKey Exists!")  # noqa
+    except Exception:
+        sql_query = """
+            CREATE OR REPLACE VIEW "MonthlyGlobalSpendPerKey" AS 
+            SELECT
+            DATE("startTime") AS date, 
+            SUM("spend") AS spend,
+            api_key as api_key
+            FROM 
+            "LiteLLM_SpendLogs" 
+            WHERE 
+            "startTime" >= (CURRENT_DATE - INTERVAL '30 days')
+            GROUP BY 
+            DATE("startTime"),
+            api_key;
+        """
+        await db.execute_raw(query=sql_query)
+
+        print("MonthlyGlobalSpendPerKey Created!")  # noqa
+    try:
+        await db.query_raw(
+            """SELECT 1 FROM "MonthlyGlobalSpendPerUserPerKey" LIMIT 1"""
+        )
+        print("MonthlyGlobalSpendPerUserPerKey Exists!")  # noqa
+    except Exception:
+        sql_query = """
+            CREATE OR REPLACE VIEW "MonthlyGlobalSpendPerUserPerKey" AS 
+            SELECT
+            DATE("startTime") AS date, 
+            SUM("spend") AS spend,
+            api_key as api_key,
+            "user" as "user"
+            FROM 
+            "LiteLLM_SpendLogs" 
+            WHERE 
+            "startTime" >= (CURRENT_DATE - INTERVAL '30 days')
+            GROUP BY 
+            DATE("startTime"),
+            "user",
+            api_key;
+        """
+        await db.execute_raw(query=sql_query)
+
+        print("MonthlyGlobalSpendPerUserPerKey Created!")  # noqa
+
+    try:
+        await db.query_raw("""SELECT 1 FROM "DailyTagSpend" LIMIT 1""")
+        print("DailyTagSpend Exists!")  # noqa
+    except Exception:
+        sql_query = """
+        CREATE OR REPLACE VIEW "DailyTagSpend" AS
+        SELECT
+            jsonb_array_elements_text(request_tags) AS individual_request_tag,
+            DATE(s."startTime") AS spend_date,
+            COUNT(*) AS log_count,
+            SUM(spend) AS total_spend
+        FROM "LiteLLM_SpendLogs" s
+        GROUP BY individual_request_tag, DATE(s."startTime");
+        """
+        await db.execute_raw(query=sql_query)
+
+        print("DailyTagSpend Created!")  # noqa
+
+    try:
+        await db.query_raw("""SELECT 1 FROM "Last30dTopEndUsersSpend" LIMIT 1""")
+        print("Last30dTopEndUsersSpend Exists!")  # noqa
+    except Exception:
+        sql_query = """
+        CREATE VIEW "Last30dTopEndUsersSpend" AS
+        SELECT end_user, COUNT(*) AS total_events, SUM(spend) AS total_spend
+        FROM "LiteLLM_SpendLogs"
+        WHERE end_user <> '' AND end_user <> user
+        AND "startTime" >= CURRENT_DATE - INTERVAL '30 days'
+        GROUP BY end_user
+        ORDER BY total_spend DESC
+        LIMIT 100;
+        """
+        await db.execute_raw(query=sql_query)
+
+        print("Last30dTopEndUsersSpend Created!")  # noqa
+
+    return
+
+
+asyncio.run(check_view_exists())

db_scripts/update_unassigned_teams.py

@@ -0,0 +1,34 @@
+from prisma import Prisma
+from litellm._logging import verbose_logger
+
+
+async def apply_db_fixes(db: Prisma):
+    """
+    Do Not Run this in production, only use it as a one-time fix
+    """
+    verbose_logger.warning(
+        "DO NOT run this in Production....Running update_unassigned_teams"
+    )
+    try:
+        sql_query = """
+            UPDATE "LiteLLM_SpendLogs"
+            SET team_id = (
+                SELECT vt.team_id
+                FROM "LiteLLM_VerificationToken" vt
+                WHERE vt.token = "LiteLLM_SpendLogs".api_key
+            )
+            WHERE team_id IS NULL
+            AND EXISTS (
+                SELECT 1
+                FROM "LiteLLM_VerificationToken" vt
+                WHERE vt.token = "LiteLLM_SpendLogs".api_key
+            );
+        """
+        response = await db.query_raw(sql_query)
+        print(
+            "Updated unassigned teams, Response=%s",
+            response,
+        )
+    except Exception as e:
+        raise Exception(f"Error apply_db_fixes: {str(e)}")
+    return

deploy/Dockerfile.ghcr_base

@@ -0,0 +1,17 @@
+# Use the provided base image
+FROM ghcr.io/berriai/litellm:main-latest
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the configuration file into the container at /app
+COPY config.yaml .
+
+# Make sure your docker/entrypoint.sh is executable
+RUN chmod +x docker/entrypoint.sh
+
+# Expose the necessary port
+EXPOSE 4000/tcp
+
+# Override the CMD instruction with your desired command and arguments
+CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"]

deploy/azure_resource_manager/azure_marketplace/createUiDefinition.json

@@ -0,0 +1,15 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#",
+    "handler": "Microsoft.Azure.CreateUIDef",
+    "version": "0.1.2-preview",
+    "parameters": {
+        "config": {
+            "isWizard": false,
+            "basics": { }
+        },
+        "basics": [ ],
+        "steps": [ ],
+        "outputs": { },
+        "resourceTypes": [ ]
+    }
+}

deploy/azure_resource_manager/azure_marketplace/mainTemplate.json

@@ -0,0 +1,63 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+      "imageName": {
+        "type": "string",
+        "defaultValue": "ghcr.io/berriai/litellm:main-latest"
+      },
+      "containerName": {
+        "type": "string",
+        "defaultValue": "litellm-container"
+      },
+      "dnsLabelName": {
+        "type": "string",
+        "defaultValue": "litellm"
+      },
+      "portNumber": {
+        "type": "int",
+        "defaultValue": 4000
+      }
+    },
+    "resources": [
+      {
+        "type": "Microsoft.ContainerInstance/containerGroups",
+        "apiVersion": "2021-03-01",
+        "name": "[parameters('containerName')]",
+        "location": "[resourceGroup().location]",
+        "properties": {
+          "containers": [
+            {
+              "name": "[parameters('containerName')]",
+              "properties": {
+                "image": "[parameters('imageName')]",
+                "resources": {
+                  "requests": {
+                    "cpu": 1,
+                    "memoryInGB": 2
+                  }
+                },
+                "ports": [
+                  {
+                    "port": "[parameters('portNumber')]"
+                  }
+                ]
+              }
+            }
+          ],
+          "osType": "Linux",
+          "restartPolicy": "Always",
+          "ipAddress": {
+            "type": "Public",
+            "ports": [
+              {
+                "protocol": "tcp",
+                "port": "[parameters('portNumber')]"
+              }
+            ],
+            "dnsNameLabel": "[parameters('dnsLabelName')]"
+          }
+        }
+      }
+    ]
+  }

deploy/azure_resource_manager/main.bicep

@@ -0,0 +1,42 @@
+param imageName string = 'ghcr.io/berriai/litellm:main-latest'
+param containerName string = 'litellm-container'
+param dnsLabelName string = 'litellm'
+param portNumber int = 4000
+
+resource containerGroupName 'Microsoft.ContainerInstance/containerGroups@2021-03-01' = {
+  name: containerName
+  location: resourceGroup().location
+  properties: {
+    containers: [
+      {
+        name: containerName
+        properties: {
+          image: imageName
+          resources: {
+            requests: {
+              cpu: 1
+              memoryInGB: 2
+            }
+          }
+          ports: [
+            {
+              port: portNumber
+            }
+          ]
+        }
+      }
+    ]
+    osType: 'Linux'
+    restartPolicy: 'Always'
+    ipAddress: {
+      type: 'Public'
+      ports: [
+        {
+          protocol: 'tcp'
+          port: portNumber
+        }
+      ]
+      dnsNameLabel: dnsLabelName
+    }
+  }
+}

deploy/charts/litellm-helm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deploy/charts/litellm-helm/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: postgresql
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 14.3.1
+- name: redis
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 18.19.1
+digest: sha256:8660fe6287f9941d08c0902f3f13731079b8cecd2a5da2fbc54e5b7aae4a6f62
+generated: "2024-03-10T02:28:52.275022+05:30"

deploy/charts/litellm-helm/Chart.yaml

@@ -0,0 +1,37 @@
+apiVersion: v2
+
+# We can't call ourselves just "litellm" because then we couldn't publish to the
+#  same OCI repository as the "litellm" OCI image
+name: litellm-helm
+description: Call all LLM APIs using the OpenAI format
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.3.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: v1.50.2
+
+dependencies:
+  - name: "postgresql"
+    version: ">=13.3.0"
+    repository: oci://registry-1.docker.io/bitnamicharts
+    condition: db.deployStandalone
+  - name: redis
+    version: ">=18.0.0"
+    repository: oci://registry-1.docker.io/bitnamicharts 
+    condition: redis.enabled

deploy/charts/litellm-helm/README.md

@@ -0,0 +1,129 @@
+# Helm Chart for LiteLLM
+
+> [!IMPORTANT]
+> This is community maintained, Please make an issue if you run into a bug
+> We recommend using [Docker or Kubernetes for production deployments](https://docs.litellm.ai/docs/proxy/prod)
+
+## Prerequisites
+
+- Kubernetes 1.21+
+- Helm 3.8.0+
+
+If `db.deployStandalone` is used:
+- PV provisioner support in the underlying infrastructure
+
+If `db.useStackgresOperator` is used (not yet implemented):
+- The Stackgres Operator must already be installed in the Kubernetes Cluster.  This chart will **not** install the operator if it is missing.
+
+## Parameters
+
+### LiteLLM Proxy Deployment Settings
+
+| Name                                                       | Description                                                                                                                                                                           | Value |
+| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| `replicaCount`                                             | The number of LiteLLM Proxy pods to be deployed                                                                                                                                       | `1`  |
+| `masterkey`                                                | The Master API Key for LiteLLM.  If not specified, a random key is generated.                                                                                                         | N/A  |
+| `environmentSecrets`                                       | An optional array of Secret object names.  The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
+| `environmentConfigMaps`                                       | An optional array of ConfigMap object names.  The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
+| `image.repository`                                         | LiteLLM Proxy image repository                                                                                                                                                        | `ghcr.io/berriai/litellm`  |
+| `image.pullPolicy`                                         | LiteLLM Proxy image pull policy                                                                                                                                                       | `IfNotPresent`  |
+| `image.tag`                                                | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published.                                                                             | `""`  |
+| `imagePullSecrets`                                         | Registry credentials for the LiteLLM and initContainer images.                                                                                                                        | `[]`  |
+| `serviceAccount.create`                                    | Whether or not to create a Kubernetes Service Account for this deployment.  The default is `false` because LiteLLM has no need to access the Kubernetes API.                          | `false`  |
+| `service.type`                                             | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                      | `ClusterIP`  |
+| `service.port`                                             | TCP port that the Kubernetes Service will listen on.  Also the TCP port within the Pod that the proxy will listen on.                                                                 | `4000`  |
+| `ingress.*`                                                | See [values.yaml](./values.yaml) for example settings                                                                                                                                 | N/A  |
+| `proxy_config.*`                                           | See [values.yaml](./values.yaml) for default settings.  See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples.                            | N/A  |
+| `extraContainers[]`                                        | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.                                                                                             | `[]`  |
+
+#### Example `environmentSecrets` Secret 
+
+```
+apiVersion: v1
+kind: Secret
+metadata:
+  name: litellm-envsecrets
+data:
+  AZURE_OPENAI_API_KEY: TXlTZWN1cmVLM3k=
+type: Opaque
+```
+
+### Database Settings
+| Name                                                       | Description                                                                                                                                                                           | Value |
+| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| `db.useExisting`                                           | Use an existing Postgres database.  A Kubernetes Secret object must exist that contains credentials for connecting to the database.  An example secret object definition is provided below.  | `false`  |
+| `db.endpoint`                                              | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to.                                                                         | `localhost`  |
+| `db.database`                                              | If `db.useExisting` is `true`, the name of the existing database to connect to.                                                                                                       | `litellm`  |
+| `db.url`                                              | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value.                                                                                                       | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)`  |
+| `db.secret.name`                                           | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials.                                                                                           | `postgres`  |
+| `db.secret.usernameKey`                                    | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance.                                | `username`  |
+| `db.secret.passwordKey`                                    | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user.                                               | `password`  |
+| `db.useStackgresOperator`                                  | Not yet implemented.                                                                                                                                                                  | `false`  |
+| `db.deployStandalone`                                      | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart.  This is useful for getting started but doesn't provide HA or (by default) data backups.  | `true`  |
+| `postgresql.*`                                             | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart.   See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details.  See [values.yaml](./values.yaml) for the default configuration.  | See [values.yaml](./values.yaml) |
+| `postgresql.auth.*`                                        | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                            | `NoTaGrEaTpAsSwOrD`  |
+
+#### Example Postgres `db.useExisting` Secret
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres
+data:
+  # Password for the "postgres" user
+  postgres-password: <some secure password, base64 encoded>
+  username: litellm
+  password: <some secure password, base64 encoded>
+type: Opaque
+```
+
+#### Examples for `environmentSecrets` and `environemntConfigMaps`
+
+```yaml
+# Use config map for not-secret configuration data
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: litellm-env-configmap
+data:
+  SOME_KEY: someValue
+  ANOTHER_KEY: anotherValue
+```
+
+```yaml
+# Use secrets for things which are actually secret like API keys, credentials, etc
+# Base64 encode the values stored in a Kubernetes Secret: $ pbpaste | base64 | pbcopy
+# The --decode flag is convenient: $ pbpaste | base64 --decode
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: litellm-env-secret
+type: Opaque
+data:
+  SOME_PASSWORD: cDZbUGVXeU5e0ZW    # base64 encoded
+  ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded
+```
+
+Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472732157221e04c6b15e3b3f094)
+
+## Accessing the Admin UI
+When browsing to the URL published per the settings in `ingress.*`, you will
+be prompted for **Admin Configuration**.  The **Proxy Endpoint** is the internal
+(from the `litellm` pod's perspective) URL published by the `<RELEASE>-litellm`
+Kubernetes Service.  If the deployment uses the default settings for this
+service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`.
+
+The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
+was not provided to the helm command line, the `masterkey` is a randomly
+generated string stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
+
+```bash
+kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"
+```
+
+## Admin UI Limitations
+At the time of writing, the Admin UI is unable to add models.  This is because
+it would need to update the `config.yaml` file which is a exposed ConfigMap, and
+therefore, read-only.  This is a limitation of this helm chart, not the Admin UI
+itself.