Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@itssimon
itssimon committed Nov 17, 2024
1 parent c8d5469 commit 0561cc0
Show file tree
Hide file tree
Showing 6 changed files with 22 additions and 6 deletions.
12 changes: 6 additions & 6 deletions apitally/client/request_logging.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import re
import tempfile
import threading
import time
from collections import deque
from contextlib import suppress
from dataclasses import dataclass, field
Expand All @@ -19,9 +18,10 @@

logger = get_logger(__name__)

MAX_BODY_SIZE = 100_000 # 100 KB (uncompressed)
MAX_FILE_SIZE = 2_000_000 # 2 MB (compressed)
MAX_BODY_SIZE = 50_000 # 50 KB (uncompressed)
MAX_FILE_SIZE = 1_000_000 # 1 MB (compressed)
MAX_REQUESTS_IN_DEQUE = 100 # Written to file every second, so limits logging to 100 rps
MAX_FILES_IN_DEQUE = 50
BODY_TOO_LARGE = b"<body too large>"
BODY_MASKED = b"<masked>"
MASKED = "******"
Expand Down Expand Up @@ -84,6 +84,7 @@ class RequestLoggingConfig:


class RequestDict(TypedDict):
timestamp: float
method: str
path: Optional[str]
url: str
Expand Down Expand Up @@ -204,7 +205,6 @@ def log_request(self, request: RequestDict, response: ResponseDict) -> None:
response["body"] = BODY_TOO_LARGE

Check warning on line 205 in apitally/client/request_logging.py

View check run for this annotation

Codecov / codecov/patch

apitally/client/request_logging.py#L205 

Added line #L205 was not covered by tests

item = {
"time_ns": time.time_ns() - response["response_time"] * 1_000_000_000,
"uuid": str(uuid4()),
"request": _skip_empty_values(request),
"response": _skip_empty_values(response),
Expand Down Expand Up @@ -244,7 +244,7 @@ def rotate_file(self) -> None:
def maybe_rotate_file(self) -> None:
if self.current_file_size > MAX_FILE_SIZE:
self.rotate_file()

Check warning on line 246 in apitally/client/request_logging.py

View check run for this annotation

Codecov / codecov/patch

apitally/client/request_logging.py#L246 

Added line #L246 was not covered by tests
while len(self.file_deque) > 50:
while len(self.file_deque) > MAX_FILES_IN_DEQUE:
file = self.file_deque.popleft()
file.delete()

Check warning on line 249 in apitally/client/request_logging.py

View check run for this annotation

Codecov / codecov/patch

apitally/client/request_logging.py#L248-L249 

Added lines #L248 - L249 were not covered by tests

Expand Down Expand Up @@ -293,7 +293,7 @@ def _has_supported_content_type(headers: List[Tuple[str, str]]) -> bool:

def _check_writable_fs():
try:
with tempfile.TemporaryFile():
with tempfile.NamedTemporaryFile():
return True
except (IOError, OSError): # pragma: no cover
logger.error("Unable to create temporary file for request logging")
Expand Down
2 changes: 2 additions & 0 deletions apitally/django.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ def configure(

def __call__(self, request: HttpRequest) -> HttpResponse:
if request.method is not None and request.method != "OPTIONS":
timestamp = time.time()
request_size = parse_int(request.headers.get("Content-Length"))
request_body = b""
if self.capture_request_body:
Expand Down Expand Up @@ -189,6 +190,7 @@ def __call__(self, request: HttpRequest) -> HttpResponse:
if self.client.request_logger.enabled:
self.client.request_logger.log_request(
request={
"timestamp": timestamp,
"method": request.method,
"path": path,
"url": request.build_absolute_uri(),
Expand Down
4 changes: 4 additions & 0 deletions apitally/flask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ def _delayed_set_startup_data(self, app_version: Optional[str] = None, openapi_u
self.client.set_startup_data(data)

def __call__(self, environ: WSGIEnvironment, start_response: StartResponse) -> Iterable[bytes]:
timestamp = time.time()
response_headers = Headers([])
status_code = 0

Expand Down Expand Up @@ -105,6 +106,7 @@ def catching_start_response(status: str, headers: List[Tuple[str, str]], exc_inf
break

Check warning on line 106 in apitally/flask.py

View check run for this annotation

Codecov / codecov/patch

apitally/flask.py#L105-L106 

Added lines #L105 - L106 were not covered by tests

self.add_request(
timestamp=timestamp,
request=request,
request_body=request_body,
status_code=status_code,
Expand All @@ -125,6 +127,7 @@ def handle_exception(e: Exception) -> Response:

def add_request(
self,
timestamp: float,
request: Request,
request_body: bytes,
status_code: int,
Expand Down Expand Up @@ -160,6 +163,7 @@ def add_request(
if self.client.request_logger.enabled:
self.client.request_logger.log_request(
request={
"timestamp": timestamp,
"method": request.method,
"path": path,
"url": request.url,
Expand Down
4 changes: 4 additions & 0 deletions apitally/litestar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ def after_exception(self, exception: Exception, scope: Scope) -> None:
def middleware_factory(self, app: ASGIApp) -> ASGIApp:
async def middleware(scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] == "http" and scope["method"] != "OPTIONS":
timestamp = time.time()
request = Request(scope)
request_size = parse_int(request.headers.get("Content-Length"))
request_body = b""
Expand Down Expand Up @@ -134,6 +135,7 @@ async def send_wrapper(message: Message) -> None:

await app(scope, receive_wrapper, send_wrapper)
self.add_request(
timestamp=timestamp,
request=request,
request_body=request_body if not request_body_too_large else BODY_TOO_LARGE,
request_size=request_size,
Expand All @@ -150,6 +152,7 @@ async def send_wrapper(message: Message) -> None:

def add_request(
self,
timestamp: float,
request: Request,
request_body: bytes,
request_size: Optional[int],
Expand Down Expand Up @@ -217,6 +220,7 @@ def add_request(
if self.client.request_logger.enabled:
self.client.request_logger.log_request(
request={
"timestamp": timestamp,
"method": request.method,
"path": path,
"url": str(request.url),
Expand Down
4 changes: 4 additions & 0 deletions apitally/starlette.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ async def _delayed_set_startup_data(

async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] == "http" and scope["method"] != "OPTIONS":
timestamp = time.time()
request = Request(scope)
request_size = parse_int(request.headers.get("Content-Length"))
request_body = b""
Expand Down Expand Up @@ -130,6 +131,7 @@ async def send_wrapper(message: Message) -> None:
if response_time is None:
response_time = time.perf_counter() - start_time
self.add_request(
timestamp=timestamp,
request=request,
request_body=request_body if not request_body_too_large else BODY_TOO_LARGE,
request_size=request_size,
Expand All @@ -145,6 +147,7 @@ async def send_wrapper(message: Message) -> None:

def add_request(
self,
timestamp: float,
request: Request,
request_body: bytes,
request_size: Optional[int],
Expand Down Expand Up @@ -195,6 +198,7 @@ def add_request(
if self.client.request_logger.enabled:
self.client.request_logger.log_request(
request={
"timestamp": timestamp,
"method": request.method,
"path": path,
"url": str(request.url),
Expand Down
2 changes: 2 additions & 0 deletions tests/test_client_request_logging.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import base64
import gzip
import json
import time
from typing import TYPE_CHECKING, Iterator, Optional
from urllib.parse import quote

Expand Down Expand Up @@ -33,6 +34,7 @@ def request_logger() -> Iterator[RequestLogger]:
@pytest.fixture()
def request_dict() -> RequestDict:
return {
"timestamp": time.time(),
"method": "GET",
"path": "/test",
"url": "http://localhost:8000/test?foo=bar",
Expand Down

0 comments on commit 0561cc0

Please sign in to comment.