feat: support auditing

Author: iziang

aperag/api/components/schemas/audit.yaml

@@ -0,0 +1,89 @@
+auditLog:
+  type: object
+  description: Audit log entry
+  properties:
+    id:
+      type: string
+      description: Audit log ID
+    user_id:
+      type: string
+      nullable: true
+      description: User ID who performed the action
+    username:
+      type: string
+      nullable: true
+      description: Username for display
+    resource_type:
+      type: string
+      enum: [collection, document, bot, chat, message, api_key, llm_provider, llm_provider_model, model_service_provider, user, config]
+      nullable: true
+      description: Type of resource
+    resource_id:
+      type: string
+      nullable: true
+      description: ID of the resource (extracted at query time)
+    api_name:
+      type: string
+      description: API operation name
+    http_method:
+      type: string
+      description: HTTP method (POST, PUT, DELETE)
+    path:
+      type: string
+      description: API path
+    status_code:
+      type: integer
+      nullable: true
+      description: HTTP status code
+    start_time:
+      type: integer
+      format: int64
+      description: Request start time (milliseconds since epoch)
+    end_time:
+      type: integer
+      format: int64
+      nullable: true
+      description: Request end time (milliseconds since epoch)
+    duration_ms:
+      type: integer
+      nullable: true
+      description: Request duration in milliseconds (calculated)
+    request_data:
+      type: string
+      nullable: true
+      description: Request data (JSON string)
+    response_data:
+      type: string
+      nullable: true
+      description: Response data (JSON string)
+    error_message:
+      type: string
+      nullable: true
+      description: Error message if failed
+    ip_address:
+      type: string
+      nullable: true
+      description: Client IP address
+    user_agent:
+      type: string
+      nullable: true
+      description: User agent string
+    request_id:
+      type: string
+      description: Request ID for tracking
+    created:
+      type: string
+      format: date-time
+      description: Created timestamp
+
+auditLogList:
+  type: object
+  description: List of audit logs
+  properties:
+    items:
+      type: array
+      description: Audit log entries
+      items:
+        $ref: '#/auditLog'
+
+ 

aperag/api/openapi.yaml

@@ -88,6 +88,12 @@ paths:
   /prompt-templates:
     $ref: './paths/prompt_templates.yaml#/promptTemplates'
 
+  # audit
+  /audit-logs:
+    $ref: './paths/audit.yaml#/audit_logs'
+  /audit-logs/{audit_id}:
+    $ref: './paths/audit.yaml#/audit_log_detail'
+  
   # users
   /invite:
     $ref: './paths/auth.yaml#/invite'

aperag/api/paths/audit.yaml

@@ -0,0 +1,111 @@
+audit_logs:
+  get:
+    tags:
+      - audit
+    summary: List audit logs
+    description: List audit logs with filtering options
+    operationId: list_audit_logs
+    parameters:
+      - name: user_id
+        in: query
+        required: false
+        schema:
+          type: string
+        description: Filter by user ID
+      - name: username
+        in: query
+        required: false
+        schema:
+          type: string
+        description: Filter by username
+      - name: resource_type
+        in: query
+        required: false
+        schema:
+          type: string
+          enum: [collection, document, bot, chat, message, api_key, llm_provider, llm_provider_model, model_service_provider, user, config]
+        description: Filter by resource type
+      - name: resource_id
+        in: query
+        required: false
+        schema:
+          type: string
+        description: Filter by resource ID
+      - name: api_name
+        in: query
+        required: false
+        schema:
+          type: string
+        description: Filter by API name
+      - name: http_method
+        in: query
+        required: false
+        schema:
+          type: string
+          enum: [POST, PUT, DELETE]
+        description: Filter by HTTP method
+      - name: status_code
+        in: query
+        required: false
+        schema:
+          type: integer
+        description: Filter by status code
+      - name: start_date
+        in: query
+        required: false
+        schema:
+          type: string
+          format: date-time
+        description: Filter by start date
+      - name: end_date
+        in: query
+        required: false
+        schema:
+          type: string
+          format: date-time
+        description: Filter by end date
+      - name: limit
+        in: query
+        required: false
+        schema:
+          type: integer
+          maximum: 5000
+          default: 1000
+        description: Maximum number of records
+    responses:
+      "200":
+        description: Audit logs retrieved successfully
+        content:
+          application/json:
+            schema:
+              $ref: "../components/schemas/audit.yaml#/auditLogList"
+      "403":
+        description: Admin access required
+
+audit_log_detail:
+  get:
+    tags:
+      - audit
+    summary: Get audit log detail
+    description: Get a specific audit log by ID
+    operationId: get_audit_log
+    parameters:
+      - name: audit_id
+        in: path
+        required: true
+        schema:
+          type: string
+        description: Audit log ID
+    responses:
+      "200":
+        description: Audit log retrieved successfully
+        content:
+          application/json:
+            schema:
+              $ref: "../components/schemas/audit.yaml#/auditLog"
+      "403":
+        description: Admin access required
+      "404":
+        description: Audit log not found
+
+ 

aperag/app.py

@@ -16,7 +16,9 @@
 
 from aperag.exception_handlers import register_exception_handlers
 from aperag.llm.litellm_track import register_opik_llm_track
+from aperag.middleware.audit_middleware import AuditMiddleware
 from aperag.views.api_key import router as api_key_router
+from aperag.views.audit import router as audit_router
 from aperag.views.auth import router as auth_router
 from aperag.views.chat_completion import router as chat_completion_router
 from aperag.views.config import router as config_router
@@ -35,9 +37,13 @@
 
 register_opik_llm_track()
 
+# Add audit middleware - should be added before other middlewares/routers
+app.add_middleware(AuditMiddleware, enabled=True)
+
 app.include_router(auth_router, prefix="/api/v1")
 app.include_router(main_router, prefix="/api/v1")
 app.include_router(api_key_router, prefix="/api/v1")
+app.include_router(audit_router, prefix="/api/v1")  # Add audit router
 app.include_router(flow_router, prefix="/api/v1")
 app.include_router(llm_router, prefix="/api/v1")
 app.include_router(chat_completion_router, prefix="/v1")

aperag/db/models.py

@@ -29,6 +29,9 @@
     Text,
     UniqueConstraint,
     select,
+    Index,
+    Float,
+    func,
 )
 from sqlalchemy import Enum as SQLEnum
 from sqlalchemy.ext.declarative import declarative_base
@@ -746,3 +749,59 @@ def update_spec(self, desired_state: IndexDesiredState = None, created_by: str =
             self.created_by = created_by
         self.version += 1
         self.gmt_updated = utc_now()
+
+
+class AuditResource(str, Enum):
+    """Audit resource types"""
+    COLLECTION = "collection"
+    DOCUMENT = "document"
+    BOT = "bot"
+    CHAT = "chat"
+    MESSAGE = "message"
+    API_KEY = "api_key"
+    LLM_PROVIDER = "llm_provider"
+    LLM_PROVIDER_MODEL = "llm_provider_model"
+    MODEL_SERVICE_PROVIDER = "model_service_provider"
+    USER = "user"
+    CONFIG = "config"
+
+
+class AuditLog(Base):
+    """Audit log model to track all system operations"""
+
+    __tablename__ = "audit_log"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    user_id = Column(String(36), nullable=True, comment="User ID")
+    username = Column(String(255), nullable=True, comment="Username")
+    resource_type = Column(Enum(AuditResource), nullable=True, comment="Resource type")
+    resource_id = Column(String(255), nullable=True, comment="Resource ID (extracted at query time)")
+    api_name = Column(String(255), nullable=False, comment="API operation name")
+    http_method = Column(String(10), nullable=False, comment="HTTP method (POST, PUT, DELETE)")
+    path = Column(String(512), nullable=False, comment="API path")
+    status_code = Column(Integer, nullable=True, comment="HTTP status code")
+    request_data = Column(Text, nullable=True, comment="Request data (JSON)")
+    response_data = Column(Text, nullable=True, comment="Response data (JSON)")
+    error_message = Column(Text, nullable=True, comment="Error message if failed")
+    ip_address = Column(String(45), nullable=True, comment="Client IP address")
+    user_agent = Column(String(500), nullable=True, comment="User agent string")
+    request_id = Column(String(255), nullable=False, comment="Request ID for tracking")
+    start_time = Column(BigInteger, nullable=False, comment="Request start time (milliseconds since epoch)")
+    end_time = Column(BigInteger, nullable=True, comment="Request end time (milliseconds since epoch)")
+    gmt_created = Column(DateTime(timezone=True), nullable=False, default=func.now(), comment="Created time")
+
+    # Index for better query performance
+    __table_args__ = (
+        Index("idx_audit_user_id", "user_id"),
+        Index("idx_audit_resource_type", "resource_type"),
+        Index("idx_audit_api_name", "api_name"),
+        Index("idx_audit_http_method", "http_method"),
+        Index("idx_audit_status_code", "status_code"),
+        Index("idx_audit_gmt_created", "gmt_created"),
+        Index("idx_audit_resource_id", "resource_id"),
+        Index("idx_audit_request_id", "request_id"),
+        Index("idx_audit_start_time", "start_time"),
+    )
+
+    def __repr__(self):
+        return f"<AuditLog(id={self.id}, user={self.username}, api={self.api_name}, method={self.http_method}, status={self.status_code})>"