Timeouts with opensearch
Happens occasionally, needs investigation. Perhaps we need to:
- add a retry mechanism
- put the message queue between the API and OpenSearch
- batch the queries differently
WARN:opensearch:POST https://es-lhcb-dpa.cern.ch:443/es/_bulk [status:N/A request:10.129s]
Traceback (most recent call last):
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/http_aiohttp.py", line 284, in perform_request
async with self.session.request(
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/aiohttp/client.py", line 1141, in __aenter__
self._resp = await self._coro
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/aiohttp/client.py", line 637, in _request
break
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/aiohttp/helpers.py", line 721, in __exit__
raise asyncio.TimeoutError from None
asyncio.exceptions.TimeoutError
INFO: 130.246.221.49:0 - "POST /ci/job/logs HTTP/1.1" 500 Internal Server Error
DEBUG:LbAPI.timing:Timing of .LbAPI.routers.ci.add_logs (['http_status:500', 'http_method:POST', 'time:wall']) was 10.150632619857788
DEBUG:LbAPI.timing:Timing of .LbAPI.routers.ci.add_logs (['http_status:500', 'http_method:POST', 'time:cpu']) was 0.03434799999996585
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/http_aiohttp.py", line 284, in perform_request
async with self.session.request(
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/aiohttp/client.py", line 1141, in __aenter__
self._resp = await self._coro
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/aiohttp/client.py", line 637, in _request
break
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/aiohttp/helpers.py", line 721, in __exit__
raise asyncio.TimeoutError from None
asyncio.exceptions.TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/uvicorn/protocols/http/httptools_impl.py", line 435, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/fastapi/applications.py", line 289, in __call__
await super().__call__(scope, receive, send)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/applications.py", line 122, in __call__
await self.middleware_stack(scope, receive, send)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/middleware/errors.py", line 184, in __call__
raise exc
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/middleware/errors.py", line 162, in __call__
await self.app(scope, receive, _send)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/middleware/cors.py", line 83, in __call__
await self.app(scope, receive, send)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/timing_asgi/middleware.py", line 68, in __call__
await self.app(scope, receive, send_wrapper)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
raise exc
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
await self.app(scope, receive, sender)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__
raise e
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__
await self.app(scope, receive, send)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/routing.py", line 718, in __call__
await route.handle(scope, receive, send)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/routing.py", line 276, in handle
await self.app(scope, receive, send)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/starlette/routing.py", line 66, in app
response = await func(request)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/fastapi/routing.py", line 273, in app
raw_response = await run_endpoint_function(
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/fastapi/routing.py", line 190, in run_endpoint_function
return await dependant.call(**values)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/LbAPI/routers/ci/__init__.py", line 234, in add_logs
n_actions = await async_bulk(
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/helpers.py", line 277, in async_bulk
async for ok, item in async_streaming_bulk(
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/helpers.py", line 195, in async_streaming_bulk
async for data, (ok, info) in azip(
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/helpers.py", line 125, in azip
yield tuple([await x.__anext__() for x in aiters])
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/helpers.py", line 125, in <listcomp>
yield tuple([await x.__anext__() for x in aiters])
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/helpers.py", line 100, in _process_bulk_chunk
for item in gen:
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/helpers/actions.py", line 204, in _process_bulk_chunk_error
raise error
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/helpers.py", line 84, in _process_bulk_chunk
resp = await client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs)
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/client/__init__.py", line 407, in bulk
return await self.transport.perform_request(
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/transport.py", line 412, in perform_request
raise e
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/transport.py", line 374, in perform_request
status, headers_response, data = await connection.perform_request(
File "/opt/conda/envs/lbapi/lib/python3.9/site-packages/opensearchpy/_async/http_aiohttp.py", line 316, in perform_request
raise ConnectionTimeout("TIMEOUT", str(e), e)
opensearchpy.exceptions.ConnectionTimeout: ConnectionTimeout caused by - TimeoutError()