From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 1 Nov 2025 14:49:52 +0000 (-0700) Subject: Fixhancement: truncate large logs, improve auto-scroll (#11239) X-Git-Tag: v2.19.4~15 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a0d3527d206eeb7745c4c240f1ef6089dbcf77e1;p=thirdparty%2Fpaperless-ngx.git Fixhancement: truncate large logs, improve auto-scroll (#11239) --- diff --git a/src-ui/src/app/components/admin/logs/logs.component.html b/src-ui/src/app/components/admin/logs/logs.component.html index 21df9f33bd..bdc80583f0 100644 --- a/src-ui/src/app/components/admin/logs/logs.component.html +++ b/src-ui/src/app/components/admin/logs/logs.component.html @@ -3,9 +3,23 @@ i18n-title info="Review the log files for the application and for email checking." i18n-info> -
- - +
+
+ Show + + lines +
+
+ + +
diff --git a/src-ui/src/app/components/admin/logs/logs.component.spec.ts b/src-ui/src/app/components/admin/logs/logs.component.spec.ts index 841fec44d5..728916830e 100644 --- a/src-ui/src/app/components/admin/logs/logs.component.spec.ts +++ b/src-ui/src/app/components/admin/logs/logs.component.spec.ts @@ -67,7 +67,7 @@ describe('LogsComponent', () => { }) it('should display logs with first log initially', () => { - expect(logSpy).toHaveBeenCalledWith('paperless') + expect(logSpy).toHaveBeenCalledWith('paperless', 5000) fixture.detectChanges() expect(fixture.debugElement.nativeElement.textContent).toContain( paperless_logs[0] @@ -78,7 +78,7 @@ describe('LogsComponent', () => { fixture.debugElement .queryAll(By.directive(NgbNavLink))[1] .nativeElement.dispatchEvent(new MouseEvent('click')) - expect(logSpy).toHaveBeenCalledWith('mail') + expect(logSpy).toHaveBeenCalledWith('mail', 5000) }) it('should handle error with no logs', () => { @@ -101,4 +101,13 @@ describe('LogsComponent', () => { jest.advanceTimersByTime(6000) expect(reloadSpy).toHaveBeenCalledTimes(2) }) + + it('should debounce limit changes before reloading logs', () => { + const initialCalls = reloadSpy.mock.calls.length + component.onLimitChange(6000) + jest.advanceTimersByTime(299) + expect(reloadSpy).toHaveBeenCalledTimes(initialCalls) + jest.advanceTimersByTime(1) + expect(reloadSpy).toHaveBeenCalledTimes(initialCalls + 1) + }) }) diff --git a/src-ui/src/app/components/admin/logs/logs.component.ts b/src-ui/src/app/components/admin/logs/logs.component.ts index 488f9db264..68b88265da 100644 --- a/src-ui/src/app/components/admin/logs/logs.component.ts +++ b/src-ui/src/app/components/admin/logs/logs.component.ts @@ -13,7 +13,7 @@ import { } from '@angular/core' import { FormsModule, ReactiveFormsModule } from '@angular/forms' import { NgbNavModule } from '@ng-bootstrap/ng-bootstrap' -import { filter, takeUntil, timer } from 'rxjs' +import { Subject, debounceTime, filter, takeUntil, timer } from 'rxjs' import { LogService } from 'src/app/services/rest/log.service' import { PageHeaderComponent } from '../../common/page-header/page-header.component' import { LoadingComponentWithPermissions } from '../../loading-component/loading.component' @@ -47,9 +47,17 @@ export class LogsComponent public autoRefreshEnabled: boolean = true + public limit: number = 5000 + + private readonly limitChange$ = new Subject() + @ViewChild('logContainer') logContainer: CdkVirtualScrollViewport ngOnInit(): void { + this.limitChange$ + .pipe(debounceTime(300), takeUntil(this.unsubscribeNotifier)) + .subscribe(() => this.reloadLogs()) + this.logService .list() .pipe(takeUntil(this.unsubscribeNotifier)) @@ -75,16 +83,33 @@ export class LogsComponent super.ngOnDestroy() } + onLimitChange(limit: number): void { + this.limitChange$.next(limit) + } + reloadLogs() { this.loading = true this.logService - .get(this.activeLog) + .get(this.activeLog, this.limit) .pipe(takeUntil(this.unsubscribeNotifier)) .subscribe({ next: (result) => { - this.logs = this.parseLogsWithLevel(result) this.loading = false - this.scrollToBottom() + const parsed = this.parseLogsWithLevel(result) + const hasChanges = + parsed.length !== this.logs.length || + parsed.some((log, idx) => { + const current = this.logs[idx] + return ( + !current || + current.message !== log.message || + current.level !== log.level + ) + }) + if (hasChanges) { + this.logs = parsed + this.scrollToBottom() + } }, error: () => { this.logs = [] diff --git a/src-ui/src/app/services/rest/log.service.spec.ts b/src-ui/src/app/services/rest/log.service.spec.ts index e3138b8950..7eda9c4a3e 100644 --- a/src-ui/src/app/services/rest/log.service.spec.ts +++ b/src-ui/src/app/services/rest/log.service.spec.ts @@ -49,4 +49,14 @@ describe('LogService', () => { ) expect(req.request.method).toEqual('GET') }) + + it('should pass limit param on logs get when provided', () => { + const id: string = 'mail' + const limit: number = 100 + subscription = service.get(id, limit).subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}${endpoint}/${id}/?limit=${limit}` + ) + expect(req.request.method).toEqual('GET') + }) }) diff --git a/src-ui/src/app/services/rest/log.service.ts b/src-ui/src/app/services/rest/log.service.ts index a836fa5550..d07f7cd698 100644 --- a/src-ui/src/app/services/rest/log.service.ts +++ b/src-ui/src/app/services/rest/log.service.ts @@ -1,4 +1,4 @@ -import { HttpClient } from '@angular/common/http' +import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable, inject } from '@angular/core' import { Observable } from 'rxjs' import { environment } from 'src/environments/environment' @@ -13,7 +13,13 @@ export class LogService { return this.http.get(`${environment.apiBaseUrl}logs/`) } - get(id: string): Observable { - return this.http.get(`${environment.apiBaseUrl}logs/${id}/`) + get(id: string, limit?: number): Observable { + let params = new HttpParams() + if (limit !== undefined) { + params = params.set('limit', limit.toString()) + } + return this.http.get(`${environment.apiBaseUrl}logs/${id}/`, { + params, + }) } } diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.py index 3f7b2c3853..8145e47937 100644 --- a/src/documents/tests/test_api_documents.py +++ b/src/documents/tests/test_api_documents.py @@ -2250,6 +2250,23 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertListEqual(response.data, ["test", "test2"]) + def test_get_log_with_limit(self): + log_data = "test1\ntest2\ntest3\n" + with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f: + f.write(log_data) + response = self.client.get("/api/logs/paperless/", {"limit": 2}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertListEqual(response.data, ["test2", "test3"]) + + def test_get_log_with_invalid_limit(self): + log_data = "test1\ntest2\n" + with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f: + f.write(log_data) + response = self.client.get("/api/logs/paperless/", {"limit": "abc"}) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response = self.client.get("/api/logs/paperless/", {"limit": -5}) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + def test_invalid_regex_other_algorithm(self): for endpoint in ["correspondents", "tags", "document_types"]: response = self.client.post( diff --git a/src/documents/views.py b/src/documents/views.py index 761cba4db3..ec347a5532 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -6,6 +6,7 @@ import re import tempfile import zipfile from collections import defaultdict +from collections import deque from datetime import datetime from pathlib import Path from time import mktime @@ -70,6 +71,7 @@ from rest_framework import parsers from rest_framework import serializers from rest_framework.decorators import action from rest_framework.exceptions import NotFound +from rest_framework.exceptions import ValidationError from rest_framework.filters import OrderingFilter from rest_framework.filters import SearchFilter from rest_framework.generics import GenericAPIView @@ -1363,6 +1365,13 @@ class UnifiedSearchViewSet(DocumentViewSet): type=OpenApiTypes.STR, location=OpenApiParameter.PATH, ), + OpenApiParameter( + name="limit", + type=OpenApiTypes.INT, + location=OpenApiParameter.QUERY, + description="Return only the last N entries from the log file", + required=False, + ), ], responses={ (200, "application/json"): serializers.ListSerializer( @@ -1394,8 +1403,22 @@ class LogViewSet(ViewSet): if not log_file.is_file(): raise Http404 + limit_param = request.query_params.get("limit") + if limit_param is not None: + try: + limit = int(limit_param) + except (TypeError, ValueError): + raise ValidationError({"limit": "Must be a positive integer"}) + if limit < 1: + raise ValidationError({"limit": "Must be a positive integer"}) + else: + limit = None + with log_file.open() as f: - lines = [line.rstrip() for line in f.readlines()] + if limit is None: + lines = [line.rstrip() for line in f.readlines()] + else: + lines = [line.rstrip() for line in deque(f, maxlen=limit)] return Response(lines)