[Bug]: Ollama as custom provider does not default to sync by default

[shanbady]

What happened?

When using ollama as an embedding provider via

from litellm import embedding
embedding(
    model='mxbai-embed-large',
    input=['test'],
    api_base="http://localhost:11434",
    custom_llm_provider='ollama',
).to_dict()

Appears to work fine on version 1.55.2

On version 1.55.7 however, I get an async exception. It appears to be defaulting to the async embedding call

Relevant log output

RuntimeError                              Traceback (most recent call last)
File /opt/venv/lib/python3.12/site-packages/litellm/main.py:3590, in embedding(model, input, dimensions, encoding_format, timeout, api_base, api_version, api_key, api_type, caching, user, custom_llm_provider, litellm_call_id, logger_fn, **kwargs)
   3585     ollama_embeddings_fn = (
   3586         ollama.ollama_aembeddings
   3587         if aembedding is True
   3588         else ollama.ollama_embeddings
   3589     )
-> 3590     response = ollama_embeddings_fn(  # type: ignore
   3591         api_base=api_base,
   3592         model=model,
   3593         prompts=input,
   3594         encoding=encoding,
   3595         logging_obj=logging,
   3596         optional_params=optional_params,
   3597         model_response=EmbeddingResponse(),
   3598     )
   3599 elif custom_llm_provider == "sagemaker":

File /opt/venv/lib/python3.12/site-packages/litellm/llms/ollama/completion/handler.py:114, in ollama_embeddings(api_base, model, prompts, optional_params, model_response, logging_obj, encoding)
    105 def ollama_embeddings(
    106     api_base: str,
    107     model: str,
   (...)
    112     encoding=None,
    113 ):
--> 114     return asyncio.run(
    115         ollama_aembeddings(
    116             api_base=api_base,
    117             model=model,
    118             prompts=prompts,
    119             model_response=model_response,
    120             optional_params=optional_params,
    121             logging_obj=logging_obj,
    122             encoding=encoding,
    123         )
    124     )

File /usr/local/lib/python3.12/asyncio/runners.py:194, in run(main, debug, loop_factory)
    193 with Runner(debug=debug, loop_factory=loop_factory) as runner:
--> 194     return runner.run(main)

File /usr/local/lib/python3.12/asyncio/runners.py:118, in Runner.run(self, coro, context)
    117 try:
--> 118     return self._loop.run_until_complete(task)
    119 except exceptions.CancelledError:

File /usr/local/lib/python3.12/asyncio/base_events.py:687, in BaseEventLoop.run_until_complete(self, future)
    685     raise RuntimeError('Event loop stopped before Future completed.')
--> 687 return future.result()

File /opt/venv/lib/python3.12/site-packages/litellm/llms/ollama/completion/handler.py:75, in ollama_aembeddings(api_base, model, prompts, model_response, optional_params, logging_obj, encoding)
     73 output_data = []
---> 75 response = await litellm.module_level_aclient.post(url=url, json=data)
     77 response_json = await response.json()

File /opt/venv/lib/python3.12/site-packages/litellm/llms/custom_httpx/http_handler.py:225, in AsyncHTTPHandler.post(self, url, data, json, params, headers, timeout, stream)
    224 except Exception as e:
--> 225     raise e

File /opt/venv/lib/python3.12/site-packages/litellm/llms/custom_httpx/http_handler.py:180, in AsyncHTTPHandler.post(self, url, data, json, params, headers, timeout, stream)
    177 req = self.client.build_request(
    178     "POST", url, data=data, json=json, params=params, headers=headers, timeout=timeout  # type: ignore
    179 )
--> 180 response = await self.client.send(req, stream=stream)
    181 response.raise_for_status()

File /opt/venv/lib/python3.12/site-packages/sentry_sdk/integrations/httpx.py:142, in _install_httpx_async_client.<locals>.send(self, request, **kwargs)
    140             request.headers[key] = value
--> 142 rv = await real_send(self, request, **kwargs)
    144 span.set_http_status(rv.status_code)

File /opt/venv/lib/python3.12/site-packages/httpx/_client.py:1674, in AsyncClient.send(self, request, stream, auth, follow_redirects)
   1672 auth = self._build_request_auth(request, auth)
-> 1674 response = await self._send_handling_auth(
   1675     request,
   1676     auth=auth,
   1677     follow_redirects=follow_redirects,
   1678     history=[],
   1679 )
   1680 try:

File /opt/venv/lib/python3.12/site-packages/httpx/_client.py:1702, in AsyncClient._send_handling_auth(self, request, auth, follow_redirects, history)
   1701 while True:
-> 1702     response = await self._send_handling_redirects(
   1703         request,
   1704         follow_redirects=follow_redirects,
   1705         history=history,
   1706     )
   1707     try:

File /opt/venv/lib/python3.12/site-packages/httpx/_client.py:1739, in AsyncClient._send_handling_redirects(self, request, follow_redirects, history)
   1737     await hook(request)
-> 1739 response = await self._send_single_request(request)
   1740 try:

File /opt/venv/lib/python3.12/site-packages/httpx/_client.py:1776, in AsyncClient._send_single_request(self, request)
   1775 with request_context(request=request):
-> 1776     response = await transport.handle_async_request(request)
   1778 assert isinstance(response.stream, AsyncByteStream)

File /opt/venv/lib/python3.12/site-packages/httpx/_transports/default.py:377, in AsyncHTTPTransport.handle_async_request(self, request)
    376 with map_httpcore_exceptions():
--> 377     resp = await self._pool.handle_async_request(req)
    379 assert isinstance(resp.stream, typing.AsyncIterable)

File /opt/venv/lib/python3.12/site-packages/httpcore/_async/connection_pool.py:256, in AsyncConnectionPool.handle_async_request(self, request)
    255     await self._close_connections(closing)
--> 256     raise exc from None
    258 # Return the response. Note that in this case we still have to manage
    259 # the point at which the response is closed.

File /opt/venv/lib/python3.12/site-packages/httpcore/_async/connection_pool.py:229, in AsyncConnectionPool.handle_async_request(self, request)
    228     closing = self._assign_requests_to_connections()
--> 229 await self._close_connections(closing)
    231 # Wait until this request has an assigned connection.

File /opt/venv/lib/python3.12/site-packages/httpcore/_async/connection_pool.py:345, in AsyncConnectionPool._close_connections(self, closing)
    344 for connection in closing:
--> 345     await connection.aclose()

File /opt/venv/lib/python3.12/site-packages/httpcore/_async/connection.py:173, in AsyncHTTPConnection.aclose(self)
    172 async with Trace("close", logger, None, {}):
--> 173     await self._connection.aclose()

File /opt/venv/lib/python3.12/site-packages/httpcore/_async/http11.py:258, in AsyncHTTP11Connection.aclose(self)
    257 self._state = HTTPConnectionState.CLOSED
--> 258 await self._network_stream.aclose()

File /opt/venv/lib/python3.12/site-packages/httpcore/_backends/anyio.py:53, in AnyIOStream.aclose(self)
     52 async def aclose(self) -> None:
---> 53     await self._stream.aclose()

File /opt/venv/lib/python3.12/site-packages/anyio/_backends/_asyncio.py:1349, in SocketStream.aclose(self)
   1347     pass
-> 1349 self._transport.close()
   1350 await sleep(0)

File /usr/local/lib/python3.12/asyncio/selector_events.py:1210, in _SelectorSocketTransport.close(self)
   1209 self._write_ready = None
-> 1210 super().close()

File /usr/local/lib/python3.12/asyncio/selector_events.py:875, in _SelectorTransport.close(self)
    874 self._loop._remove_writer(self._sock_fd)
--> 875 self._loop.call_soon(self._call_connection_lost, None)

File /usr/local/lib/python3.12/asyncio/base_events.py:795, in BaseEventLoop.call_soon(self, callback, context, *args)
    786 """Arrange for a callback to be called as soon as possible.
    787 
    788 This operates as a FIFO queue: callbacks are called in the
   (...)
    793 the callback when it is called.
    794 """
--> 795 self._check_closed()
    796 if self._debug:

File /usr/local/lib/python3.12/asyncio/base_events.py:541, in BaseEventLoop._check_closed(self)
    540 if self._closed:
--> 541     raise RuntimeError('Event loop is closed')

RuntimeError: Event loop is closed

Are you a ML Ops Team?

No

What LiteLLM version are you on ?

1.55.7

Twitter / LinkedIn details

No response

[krrishdholakia]

ack thanks for the ticket