提交 58b7a2a7 authored 作者: caifuming's avatar caifuming

5/28号 写了一部分后端

上级 f7ad0a94
# ===========================================
# Application Settings
# ===========================================
APP_NAME=PLabel
APP_ENV=development # development, staging, production
APP_DEBUG=true
APP_SECRET_KEY=change-this-to-a-super-secret-key-in-production
# Server
HOST=0.0.0.0
PORT=8000
# ===========================================
# Database Settings
# ===========================================
# PostgreSQL connection string format:
# postgresql://username:password@host:port/database
DATABASE_URL=postgresql://plabel:plabel123@localhost:5432/plabel
# For async driver (recommended)
ASYNC_DATABASE_URL=postgresql+asyncpg://plabel:plabel123@localhost:5432/plabel
# ===========================================
# Redis Settings
# ===========================================
REDIS_HOST=localhost
REDIS_PORT=6379
REDIS_DB=0
REDIS_PASSWORD=
# Redis connection URL
REDIS_URL=redis://localhost:6379/0
# Celery broker URL
CELERY_BROKER_URL=redis://localhost:6379/1
CELERY_RESULT_BACKEND=redis://localhost:6379/2
# ===========================================
# MinIO / S3 Settings
# ===========================================
# MinIO Endpoint (for local development)
MINIO_ENDPOINT=localhost:9009
MINIO_ACCESS_KEY=plabel_access_key
MINIO_SECRET_KEY=plabel_secret_key_change_this
MINIO_BUCKET=plabel-bucket
MINIO_SECURE=false # true for HTTPS, false for HTTP
MINIO_REGION=us-east-1
# AWS S3 (alternative to MinIO)
# AWS_ACCESS_KEY_ID=your_access_key
# AWS_SECRET_ACCESS_KEY=your_secret_key
# AWS_REGION=us-east-1
# S3_BUCKET=your-bucket-name
# ===========================================
# JWT Settings
# ===========================================
JWT_SECRET_KEY=your-super-secret-jwt-key-change-this-in-production
JWT_ALGORITHM=HS256
ACCESS_TOKEN_EXPIRE_MINUTES=1440 # 24 hours
REFRESH_TOKEN_EXPIRE_DAYS=7
# ===========================================
# File Upload Settings
# ===========================================
# Max file sizes in MB
MAX_IMAGE_SIZE_MB=50
MAX_VIDEO_SIZE_MB=500
MAX_AUDIO_SIZE_MB=100
MAX_TEXT_SIZE_MB=10
# Allowed file types (comma-separated MIME types)
ALLOWED_IMAGE_TYPES=image/jpeg,image/png,image/webp,image/bmp
ALLOWED_VIDEO_TYPES=video/mp4,video/webm,video/quicktime
ALLOWED_AUDIO_TYPES=audio/mpeg,audio/wav,audio/ogg,audio/m4a
ALLOWED_TEXT_TYPES=text/plain,text/markdown,application/json
# Upload directory
UPLOAD_DIR=./data/uploads
THUMBNAIL_DIR=./data/thumbnails
EXPORT_DIR=./data/exports
# ===========================================
# AI Model Settings
# ===========================================
# Device for AI inference
AI_MODEL_DEVICE=cpu # cpu or cuda
# Model cache directory
AI_MODEL_CACHE_DIR=./models
# GPU device ID (if using multiple GPUs)
CUDA_VISIBLE_DEVICES=0
# ===========================================
# Celery Settings
# ===========================================
# Task queue names
CELERY_TASK_QUEUE_DEFAULT=default
CELERY_TASK_QUEUE_GPU=gpu_tasks
CELERY_TASK_QUEUE_CPU=cpu_tasks
# Task settings
CELERY_TASK_ACKS_LATE=true
CELERY_TASK_REJECT_ON_WORKER_LOST=true
CELERY_WORKER_PREFETCH_MULTIPLIER=1
# ===========================================
# CORS Settings
# ===========================================
# Comma-separated list of allowed origins
CORS_ORIGINS=http://localhost:3000,http://localhost:8000,http://127.0.0.1:3000,http://127.0.0.1:8000
# ===========================================
# Email Settings (Optional)
# ===========================================
# SMTP configuration for sending emails
SMTP_HOST=smtp.example.com
SMTP_PORT=587
SMTP_USER=your_email@example.com
SMTP_PASSWORD=your_email_password
SMTP_FROM=PLabel <noreply@plabel.com>
SMTP_USE_TLS=true
# ===========================================
# Logging Settings
# ===========================================
LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR, CRITICAL
LOG_FILE=./logs/app.log
LOG_ROTATION=100 MB
LOG_RETENTION=30 days
# Byte-compiled / optimized
__pycache__/
*.py[cod]
*.so
.Python
venv/
.venv/
.env
.env.local
# Database
*.db
*.sqlite
# Logs & Data
logs/
data/uploads/
data/thumbnails/
data/exports/
*.log
# AI Models (large files, download on demand)
models/vision/
models/audio/
models/text/
# IDE
.idea/
.vscode/
*.swp
*.swo
*.swn
# Testing
.pytest_cache/
.coverage
htmlcov/
*.cover
# Docker
.docker/
# OS
.DS_Store
Thumbs.db
# Secrets (never commit)
*.pem
*.key
credentials.json
secrets/
FROM python:3.11-slim
WORKDIR /app
# Install system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential libpq-dev libffi-dev libssl-dev ffmpeg \
libsm6 libxext6 libxrender-dev libgomp1 git wget curl \
&& rm -rf /var/lib/apt/lists/*
# Copy requirements first for better caching
COPY requirements.txt requirements-ai.txt* /app/
RUN pip install --no-cache-dir -r requirements.txt
# Copy application code
COPY . /app/
# Create necessary directories
RUN mkdir -p /app/logs /app/data/uploads /app/data/thumbnails /app/data/exports
# Expose port
EXPOSE 8000
# Run the application
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04
WORKDIR /app
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
python3.11 python3.11-dev python3-pip \
build-essential libpq-dev libffi-dev libssl-dev ffmpeg \
libsm6 libxext6 libxrender-dev libgomp1 git wget curl \
&& rm -rf /var/lib/apt/lists/*
COPY requirements-ai.txt /app/
RUN pip install --no-cache-dir -r requirements-ai.txt
COPY . /app/
RUN mkdir -p /app/logs /app/data/uploads /app/data/thumbnails /app/data/exports /app/models
EXPOSE 8000
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
.PHONY: help install dev run test test-cov lint db-migrate db-seed docker-up docker-down clean
help:
@echo "PLabel Backend - Available commands:"
@echo " make install - Install dependencies"
@echo " make dev - Run development server"
@echo " make test - Run tests"
@echo " make test-cov - Run tests with coverage"
@echo " make lint - Run linter"
@echo " make db-migrate - Run database migrations"
@echo " make db-seed - Seed database"
@echo " make docker-up - Start all services"
@echo " make docker-down - Stop all services"
@echo " make clean - Clean cache files"
install:
pip install -r requirements.txt
pip install -r requirements-ai.txt
dev:
uvicorn app.main:app --reload --host 0.0.0.0 --port 8000
test:
pytest
test-cov:
pytest --cov=app tests/ --cov-report=html
lint:
ruff check app/
db-migrate:
alembic upgrade head
db-seed:
python -m app.utils.seed_data
docker-up:
docker-compose up -d
docker-down:
docker-compose down
clean:
find . -type d -name __pycache__ -exec rm -rf {} +
find . -type f -name "*.pyc" -delete
rm -rf .pytest_cache .coverage htmlcov
# PLabel Backend - FastAPI Backend for Data Annotation Platform
## 环境要求
- Python 3.10+
- PostgreSQL 15+
- Redis 7+
- Docker & Docker Compose (推荐)
- NVIDIA GPU (可选,用于 AI 推理)
## 快速开始
### 方式一: Docker 部署 (推荐)
```bash
# 复制环境变量配置
cp .env.example .env
# 启动所有服务
docker-compose up -d
# 查看日志
docker-compose logs -f backend
# 停止服务
docker-compose down
```
### 方式二: 本地开发
```bash
# 1. 创建虚拟环境
python -m venv venv
source venv/bin/activate # Linux/Mac
# 或
.\venv\Scripts\activate # Windows
# 2. 安装依赖
pip install -r requirements.txt
# 3. 配置环境变量
cp .env.example .env
# 编辑 .env 文件,配置数据库连接等
# 4. 初始化数据库
alembic upgrade head
# 5. 创建超级用户
python -m app.utils.create_superuser
# 6. 启动开发服务器
uvicorn app.main:app --reload --host 0.0.0.0 --port 8000
```
## 环境变量配置
请参考 `.env.example` 文件配置以下环境变量:
- `DATABASE_URL` - PostgreSQL 连接字符串
- `REDIS_URL` - Redis 连接字符串
- `MINIO_ENDPOINT` - MinIO 服务地址
- `MINIO_ACCESS_KEY` / `MINIO_SECRET_KEY` - MinIO 凭证
- `JWT_SECRET_KEY` - JWT 密钥
## API 文档
启动服务后访问:
- Swagger UI: http://localhost:8000/docs
- ReDoc: http://localhost:8000/redoc
- OpenAPI JSON: http://localhost:8000/openapi.json
## 项目结构
```
backend/
├── app/
│ ├── api/v1/ # API 路由
│ ├── core/ # 核心配置
│ ├── models/ # SQLAlchemy 模型
│ ├── schemas/ # Pydantic schemas
│ ├── services/ # 业务逻辑
│ └── utils/ # 工具函数
├── alembic/ # 数据库迁移
├── tests/ # 测试文件
├── models/ # AI 模型文件
├── data/ # 上传数据目录
├── Dockerfile
├── docker-compose.yml
└── requirements.txt
```
## Celery 任务
```bash
# 启动 Celery Worker
celery -A app.celery worker -l info
# 启动 Flower (任务监控)
celery -A app.celery flower --port=5555
```
## 测试
```bash
# 运行所有测试
pytest
# 运行带覆盖率
pytest --cov=app tests/
```
## 默认账号
- 管理员: admin@plabel.com / admin123
- 普通用户: user@plabel.com / user123
- 标注员: annotator@plabel.com / annotator123
- 审核员: reviewer@plabel.com / reviewer123
## 技术栈
- **框架**: FastAPI 0.110+
- **数据库**: PostgreSQL 15+ / SQLAlchemy 2.0
- **缓存**: Redis 7+
- **任务队列**: Celery 5.x
- **文件存储**: MinIO (S3 兼容)
- **认证**: JWT (python-jose)
- **验证**: Pydantic v2
## License
MIT License
[alembic]
script_location = alembic
prepend_sys_path = .
version_path_separator = os
sqlalchemy.url = postgresql://plabel:plabel123@localhost:5432/plabel
[post_write_hooks]
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
"""
Alembic migration environment configuration
"""
from logging.config import fileConfig
from alembic import context
from sqlalchemy import pool
from sqlalchemy import engine_from_config
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.core.config import settings
from app.core.database import Base
from app.models import * # noqa: F401,F403
config = context.config
config.set_main_option("sqlalchemy.url", settings.database_url)
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode"""
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
"""
PLabel Backend Application
FastAPI application for data annotation platform
"""
__version__ = "1.0.0"
"""
API dependencies: authentication, authorization, pagination
"""
from functools import wraps
from typing import Annotated, Callable
from fastapi import Depends, HTTPException, Path, Query, Request, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
from app.core.database import get_async_session
from app.core.security import decode_token
from app.models.user import User
security = HTTPBearer()
async def get_current_user(
credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)],
db: AsyncSession = Depends(get_async_session),
) -> User:
"""Get current authenticated user from JWT token"""
credentials_exception = HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Could not validate credentials",
headers={"WWW-Authenticate": "Bearer"},
)
try:
token = credentials.credentials
payload = decode_token(token)
if payload is None:
raise credentials_exception
user_id_str = payload.get("sub")
if user_id_str is None:
raise credentials_exception
try:
user_id = int(user_id_str)
except ValueError:
raise credentials_exception
except Exception:
raise credentials_exception
result = await db.execute(select(User).where(User.id == user_id))
user = result.scalar_one_or_none()
if user is None:
raise credentials_exception
if not user.is_active:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Inactive user"
)
return user
async def get_current_active_user(
current_user: Annotated[User, Depends(get_current_user)],
) -> User:
"""Verify current user is active"""
return current_user
def require_roles(*allowed_roles: str):
"""Dependency factory to require specific roles for an endpoint"""
async def role_checker(
current_user: User = Depends(get_current_user),
) -> User:
if current_user.role not in allowed_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"Insufficient permissions. Required roles: {', '.join(allowed_roles)}"
)
return current_user
return role_checker
async def check_project_access(
user_id: int,
project_id: int,
db: AsyncSession,
required_roles: list[str],
) -> bool:
"""Check if user has required role in a project"""
return True
class PaginationParams:
"""Pagination parameters for list endpoints"""
def __init__(
self,
page: int = Query(1, ge=1, description="Page number"),
page_size: int = Query(20, ge=1, le=100, description="Items per page"),
):
self.page = page
self.page_size = page_size
self.offset = (page - 1) * page_size
def get_pagination_params(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
) -> dict:
return {
"offset": (page - 1) * page_size,
"limit": page_size,
"page": page,
"page_size": page_size,
}
"""
API v1 router
"""
from fastapi import APIRouter
from app.api.v1 import auth, users
api_router = APIRouter()
api_router.include_router(auth.router)
api_router.include_router(users.router)
"""
Authentication API routes
"""
from datetime import timedelta
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_current_user
from app.core.config import settings
from app.core.database import get_async_session
from app.core.exceptions import AuthenticationError, ConflictError, ValidationError
from app.core.security import decode_token
from app.models.user import User
from app.schemas.auth import (
ChangePasswordRequest,
LoginRequest,
LoginResponse,
RegisterRequest,
TokenResponse,
UserResponse,
)
from app.services.auth_service import AuthService
router = APIRouter(prefix="/auth", tags=["Authentication"])
def user_to_response(user: User) -> UserResponse:
"""Convert User model to UserResponse schema"""
return UserResponse(
id=str(user.id),
username=user.username,
email=user.email,
role=user.role,
avatar_url=user.avatar_url,
is_active=user.is_active,
)
@router.post("/login", response_model=LoginResponse)
async def login(
login_data: LoginRequest,
db: AsyncSession = Depends(get_async_session),
) -> LoginResponse:
"""
User login endpoint
Returns access token and user information
"""
auth_service = AuthService(db)
try:
user, access_token = await auth_service.login(login_data)
return LoginResponse(
access_token=access_token,
token_type="bearer",
expires_in=settings.access_token_expire_minutes * 60,
user=user_to_response(user),
)
except AuthenticationError as e:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=str(e),
headers={"WWW-Authenticate": "Bearer"},
)
@router.post("/register", response_model=LoginResponse, status_code=status.HTTP_201_CREATED)
async def register(
register_data: RegisterRequest,
db: AsyncSession = Depends(get_async_session),
) -> LoginResponse:
"""
User registration endpoint
Creates a new user account and returns access token
"""
auth_service = AuthService(db)
try:
user = await auth_service.register(register_data)
access_token = create_access_token(
data={
"sub": str(user.id),
"email": user.email,
"role": user.role,
}
)
return LoginResponse(
access_token=access_token,
token_type="bearer",
expires_in=settings.access_token_expire_minutes * 60,
user=user_to_response(user),
)
except ConflictError as e:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=str(e),
)
except ValidationError as e:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(e),
)
@router.post("/logout")
async def logout(current_user: User = Depends(get_current_user)) -> dict:
"""
User logout endpoint
Note: For JWT-based auth, logout is typically handled client-side
by removing the token from storage.
"""
return {"message": "Successfully logged out"}
@router.get("/me", response_model=UserResponse)
async def get_current_user_info(
current_user: User = Depends(get_current_user),
) -> UserResponse:
"""
Get current authenticated user information
"""
return user_to_response(current_user)
@router.put("/me", response_model=UserResponse)
async def update_current_user(
user_data: dict,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_async_session),
) -> UserResponse:
"""
Update current user's profile information
"""
from app.services.user_service import UserService
from app.schemas.user import UserUpdate
user_service = UserService(db)
updated_user = await user_service.update_user(
current_user.id,
UserUpdate(**{k: v for k, v in user_data.items() if k in ["username", "email", "avatar_url"]})
)
return user_to_response(updated_user)
@router.post("/change-password")
async def change_password(
password_data: ChangePasswordRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_async_session),
) -> dict:
"""
Change current user's password
"""
from app.services.user_service import UserService
user_service = UserService(db)
try:
await user_service.update_password(
current_user.id,
password_data.old_password,
password_data.new_password,
)
return {"message": "Password updated successfully"}
except PermissionDeniedError:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Current password is incorrect",
)
except ValueError as e:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(e),
)
def create_access_token(data: dict) -> str:
"""Helper function to create access token"""
from app.core.security import create_access_token as _create_access_token
return _create_access_token(data)
"""
User management API routes
"""
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_current_user, get_pagination_params
from app.core.database import get_async_session
from app.core.exceptions import NotFoundError, PermissionDeniedError
from app.models.user import User
from app.schemas.base import ResponseSchema
from app.schemas.user import UserCreate, UserResponse, UserStatsResponse, UserUpdate
from app.services.user_service import UserService
router = APIRouter(prefix="/users", tags=["Users"])
def user_to_response(user: User) -> UserResponse:
"""Convert User model to UserResponse schema"""
return UserResponse(
id=str(user.id),
username=user.username,
email=user.email,
role=user.role,
avatar_url=user.avatar_url,
is_active=user.is_active,
)
@router.get("", response_model=ResponseSchema)
async def list_users(
page: int = Query(1, ge=1, description="Page number"),
page_size: int = Query(20, ge=1, le=100, description="Items per page"),
role: str | None = Query(None, description="Filter by role"),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_async_session),
) -> ResponseSchema:
"""
Get list of users (admin only)
"""
if current_user.role != "admin":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only administrators can access user list",
)
user_service = UserService(db)
skip = (page - 1) * page_size
users, total = await user_service.get_users(skip=skip, limit=page_size, role=role)
return ResponseSchema(
code=200,
message="success",
data={
"items": [user_to_response(u) for u in users],
"total": total,
"page": page,
"page_size": page_size,
"total_pages": (total + page_size - 1) // page_size if total > 0 else 0,
}
)
@router.get("/{user_id}", response_model=ResponseSchema)
async def get_user(
user_id: int,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_async_session),
) -> ResponseSchema:
"""
Get user by ID
"""
user_service = UserService(db)
user = await user_service.get_by_id(user_id)
if not user:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="User not found",
)
return ResponseSchema(
code=200,
message="success",
data=user_to_response(user),
)
@router.post("", response_model=ResponseSchema, status_code=status.HTTP_201_CREATED)
async def create_user(
user_data: UserCreate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_async_session),
) -> ResponseSchema:
"""
Create a new user (admin only)
"""
if current_user.role != "admin":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only administrators can create users",
)
user_service = UserService(db)
try:
user = await user_service.create_user(user_data)
return ResponseSchema(
code=201,
message="User created successfully",
data=user_to_response(user),
)
except Exception as e:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=str(e),
)
@router.put("/{user_id}", response_model=ResponseSchema)
async def update_user(
user_id: int,
user_data: UserUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_async_session),
) -> ResponseSchema:
"""
Update user information (admin only for other users, users can update themselves)
"""
user_service = UserService(db)
if current_user.role != "admin" and current_user.id != user_id:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="You can only update your own profile",
)
try:
updated_user = await user_service.update_user(user_id, user_data)
return ResponseSchema(
code=200,
message="User updated successfully",
data=user_to_response(updated_user),
)
except NotFoundError:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="User not found",
)
except Exception as e:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=str(e),
)
@router.delete("/{user_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_user(
user_id: int,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_async_session),
):
"""
Delete a user (admin only)
"""
if current_user.role != "admin":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only administrators can delete users",
)
if current_user.id == user_id:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="You cannot delete your own account",
)
user_service = UserService(db)
try:
await user_service.delete_user(user_id)
except NotFoundError:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="User not found",
)
@router.get("/{user_id}/stats", response_model=ResponseSchema)
async def get_user_stats(
user_id: int,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_async_session),
) -> ResponseSchema:
"""
Get user statistics
"""
if current_user.role != "admin" and current_user.id != user_id:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="You can only view your own statistics",
)
user_service = UserService(db)
stats = await user_service.get_user_stats(user_id)
return ResponseSchema(
code=200,
message="success",
data=stats,
)
"""
PLabel Core Package
Exposes configuration, infrastructure, and shared utilities
"""
from app.core.config import settings
from app.core.database import (
AsyncSessionLocal,
Base,
SyncSessionLocal,
async_engine,
close_db,
get_async_session,
get_db_context,
get_sync_session,
init_db,
sync_engine,
)
from app.core.exceptions import (
AIInferenceError,
AlreadyExistsError,
AuthenticationError,
DatabaseError,
FileTooLargeError,
InvalidFileTypeError,
InvalidTokenError,
NotFoundError,
PermissionDeniedError,
PLabelException,
RateLimitError,
StorageError,
TaskQueueError,
ValidationError,
register_exception_handlers,
)
from app.core.logging import get_logger, setup_logging
from app.core.middleware import RequestLogMiddleware
from app.core.redis import CacheService, close_async_redis, get_async_redis, get_cache, get_redis_client
from app.core.security import create_access_token, create_refresh_token, decode_token, hash_password, verify_password
from app.core.storage import StorageService, storage_service
__all__ = [
# Config
"settings",
# Database
"Base",
"async_engine",
"sync_engine",
"AsyncSessionLocal",
"SyncSessionLocal",
"get_async_session",
"get_sync_session",
"get_db_context",
"init_db",
"close_db",
# Redis
"get_redis_client",
"get_async_redis",
"get_cache",
"CacheService",
"close_async_redis",
# Security
"hash_password",
"verify_password",
"create_access_token",
"create_refresh_token",
"decode_token",
# Storage
"StorageService",
"storage_service",
# Logging
"setup_logging",
"get_logger",
# Middleware
"RequestLogMiddleware",
# Exceptions
"PLabelException",
"NotFoundError",
"AlreadyExistsError",
"PermissionDeniedError",
"AuthenticationError",
"InvalidTokenError",
"FileTooLargeError",
"InvalidFileTypeError",
"ValidationError",
"DatabaseError",
"StorageError",
"TaskQueueError",
"AIInferenceError",
"RateLimitError",
"register_exception_handlers",
]
"""
Application configuration settings
Loads settings from environment variables using pydantic-settings
"""
from functools import lru_cache
from typing import List
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
"""Application settings loaded from environment variables"""
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=False,
extra="ignore"
)
# ===========================================
# Application Settings
# ===========================================
app_name: str = "PLabel"
app_env: str = "development"
app_debug: bool = True
app_secret_key: str = "change-this-super-secret-key"
# Server
host: str = "0.0.0.0"
port: int = 8000
# ===========================================
# Database Settings
# ===========================================
database_url: str = "postgresql://plabel:plabel123@localhost:5432/plabel"
async_database_url: str = "postgresql+asyncpg://plabel:plabel123@localhost:5432/plabel"
# ===========================================
# Redis Settings
# ===========================================
redis_host: str = "localhost"
redis_port: int = 6379
redis_db: int = 0
redis_password: str = ""
redis_url: str = "redis://localhost:6379/0"
celery_broker_url: str = "redis://localhost:6379/1"
celery_result_backend: str = "redis://localhost:6379/2"
# ===========================================
# MinIO Settings
# ===========================================
minio_endpoint: str = "localhost:9000"
minio_access_key: str = "plabel_access_key"
minio_secret_key: str = "plabel_secret_key"
minio_bucket: str = "plabel-bucket"
minio_secure: bool = False
minio_region: str = "us-east-1"
# ===========================================
# JWT Settings
# ===========================================
jwt_secret_key: str = "your-super-secret-jwt-key"
jwt_algorithm: str = "HS256"
access_token_expire_minutes: int = 1440 # 24 hours
refresh_token_expire_days: int = 7
# ===========================================
# File Upload Settings
# ===========================================
max_image_size_mb: int = 50
max_video_size_mb: int = 500
max_audio_size_mb: int = 100
max_text_size_mb: int = 10
allowed_image_types: str = "image/jpeg,image/png,image/webp,image/bmp"
allowed_video_types: str = "video/mp4,video/webm,video/quicktime"
allowed_audio_types: str = "audio/mpeg,audio/wav,audio/ogg,audio/m4a"
allowed_text_types: str = "text/plain,text/markdown,application/json"
upload_dir: str = "./data/uploads"
thumbnail_dir: str = "./data/thumbnails"
export_dir: str = "./data/exports"
# ===========================================
# AI Model Settings
# ===========================================
ai_model_device: str = "cpu"
ai_model_cache_dir: str = "./models"
cuda_visible_devices: str = "0"
# ===========================================
# CORS Settings
# ===========================================
cors_origins: str = "http://localhost:3000,http://localhost:8000"
# ===========================================
# Logging Settings
# ===========================================
log_level: str = "INFO"
log_file: str = "./logs/app.log"
# ===========================================
# Computed Properties
# ===========================================
@property
def cors_origins_list(self) -> List[str]:
return [origin.strip() for origin in self.cors_origins.split(",")]
@property
def max_image_size(self) -> int:
return self.max_image_size_mb * 1024 * 1024
@property
def max_video_size(self) -> int:
return self.max_video_size_mb * 1024 * 1024
@property
def max_audio_size(self) -> int:
return self.max_audio_size_mb * 1024 * 1024
@property
def max_text_size(self) -> int:
return self.max_text_size_mb * 1024 * 1024
@property
def allowed_image_types_list(self) -> List[str]:
return self.allowed_image_types.split(",")
@property
def allowed_video_types_list(self) -> List[str]:
return self.allowed_video_types.split(",")
@property
def allowed_audio_types_list(self) -> List[str]:
return self.allowed_audio_types.split(",")
@property
def allowed_text_types_list(self) -> List[str]:
return self.allowed_text_types.split(",")
@property
def is_production(self) -> bool:
return self.app_env == "production"
@property
def is_development(self) -> bool:
return self.app_env == "development"
@lru_cache()
def get_settings() -> Settings:
"""Get cached settings instance"""
return Settings()
# Global settings instance
settings = get_settings()
"""
Database connection and session management
"""
from contextlib import asynccontextmanager
from typing import AsyncGenerator
from sqlalchemy import create_engine
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
from app.core.config import settings
# ===========================================
# Synchronous Engine (for migrations & CLI)
# ===========================================
sync_engine = create_engine(
settings.database_url,
pool_pre_ping=True,
pool_size=10,
max_overflow=20,
echo=settings.app_debug,
)
SyncSessionLocal = sessionmaker(
autocommit=False,
autoflush=False,
bind=sync_engine,
)
# ===========================================
# Async Engine (for API endpoints)
# ===========================================
async_engine = create_async_engine(
settings.async_database_url,
pool_pre_ping=True,
pool_size=10,
max_overflow=20,
echo=settings.app_debug,
)
AsyncSessionLocal = async_sessionmaker(
bind=async_engine,
class_=AsyncSession,
expire_on_commit=False,
autocommit=False,
autoflush=False,
)
# ===========================================
# Base Model Class
# ===========================================
class Base(DeclarativeBase):
"""Base class for all SQLAlchemy models"""
pass
# ===========================================
# Dependency Functions
# ===========================================
async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
"""
Async dependency for FastAPI to get database session
Usage:
@app.get("/items")
async def get_items(db: AsyncSession = Depends(get_async_session)):
...
"""
async with AsyncSessionLocal() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
finally:
await session.close()
def get_sync_session() -> Session:
"""
Sync dependency for getting database session
Usage in CLI scripts or background tasks:
with get_sync_session() as session:
...
"""
session = SyncSessionLocal()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
@asynccontextmanager
async def get_db_context() -> AsyncGenerator[AsyncSession, None]:
"""
Async context manager for database session
Usage:
async with get_db_context() as db:
result = await db.execute(query)
"""
async with AsyncSessionLocal() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
finally:
await session.close()
# ===========================================
# Database Initialization
# ===========================================
async def init_db():
"""Initialize database tables"""
async with async_engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
async def close_db():
"""Close database connections"""
await async_engine.dispose()
"""
Custom exceptions and exception handlers for PLabel
"""
from typing import Any, Optional
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
class PLabelException(Exception):
"""Base exception for all PLabel errors"""
def __init__(
self,
message: str,
code: int = 400,
errors: Optional[list] = None,
details: Optional[Any] = None,
):
self.message = message
self.code = code
self.errors = errors or []
self.details = details
super().__init__(self.message)
class NotFoundError(PLabelException):
"""Resource not found"""
def __init__(self, resource: str, identifier: str | int):
super().__init__(
message=f"{resource} with identifier '{identifier}' not found",
code=404,
)
self.resource = resource
self.identifier = identifier
class AlreadyExistsError(PLabelException):
"""Resource already exists"""
def __init__(self, resource: str, field: str, value: str):
super().__init__(
message=f"{resource} with {field} '{value}' already exists",
code=409,
)
self.resource = resource
self.field = field
self.value = value
class ConflictError(PLabelException):
"""Resource conflict error (e.g., duplicate entry)"""
def __init__(self, message: str = "Resource already exists"):
super().__init__(message=message, code=409)
class PermissionDeniedError(PLabelException):
"""Insufficient permissions"""
def __init__(self, message: str = "You do not have permission to perform this action"):
super().__init__(message=message, code=403)
class AuthenticationError(PLabelException):
"""Authentication failed"""
def __init__(self, message: str = "Authentication failed"):
super().__init__(message=message, code=401)
class InvalidTokenError(AuthenticationError):
"""Invalid or expired token"""
def __init__(self, message: str = "Invalid or expired token"):
super().__init__(message=message)
class FileTooLargeError(PLabelException):
"""Uploaded file exceeds size limit"""
def __init__(self, filename: str, max_size: str):
super().__init__(
message=f"File '{filename}' exceeds maximum size limit of {max_size}",
code=413,
)
self.filename = filename
self.max_size = max_size
class InvalidFileTypeError(PLabelException):
"""File type not allowed"""
def __init__(self, filename: str, allowed_types: list[str]):
super().__init__(
message=f"File type of '{filename}' is not allowed. Allowed types: {', '.join(allowed_types)}",
code=415,
)
self.filename = filename
self.allowed_types = allowed_types
class ValidationError(PLabelException):
"""Request validation failed"""
def __init__(self, errors: list[dict]):
super().__init__(
message="Validation failed",
code=422,
errors=errors,
)
class DatabaseError(PLabelException):
"""Database operation failed"""
def __init__(self, message: str = "Database operation failed"):
super().__init__(message=message, code=500)
class StorageError(PLabelException):
"""File storage operation failed"""
def __init__(self, message: str = "File storage operation failed"):
super().__init__(message=message, code=500)
class TaskQueueError(PLabelException):
"""Task queue operation failed"""
def __init__(self, message: str = "Task queue operation failed"):
super().__init__(message=message, code=500)
class AIInferenceError(PLabelException):
"""AI model inference failed"""
def __init__(self, model_name: str, message: str = "AI inference failed"):
super().__init__(
message=f"{message} (model: {model_name})",
code=500,
)
self.model_name = model_name
class RateLimitError(PLabelException):
"""Rate limit exceeded"""
def __init__(self, message: str = "Rate limit exceeded. Please try again later"):
super().__init__(message=message, code=429)
def register_exception_handlers(app: FastAPI) -> None:
"""Register all exception handlers with the FastAPI app"""
@app.exception_handler(PLabelException)
async def plabel_exception_handler(
request: Request, exc: PLabelException
) -> JSONResponse:
content = {
"code": exc.code,
"message": exc.message,
}
if exc.errors:
content["errors"] = exc.errors
if exc.details:
content["details"] = exc.details
return JSONResponse(status_code=exc.code, content=content)
@app.exception_handler(NotFoundError)
async def not_found_handler(
request: Request, exc: NotFoundError
) -> JSONResponse:
return JSONResponse(
status_code=404,
content={
"code": 404,
"message": exc.message,
"errors": [{"field": "id", "message": exc.message}],
},
)
@app.exception_handler(ValidationError)
async def validation_handler(
request: Request, exc: ValidationError
) -> JSONResponse:
return JSONResponse(
status_code=422,
content={
"code": 422,
"message": exc.message,
"errors": exc.errors,
},
)
@app.exception_handler(AuthenticationError)
async def auth_error_handler(
request: Request, exc: AuthenticationError
) -> JSONResponse:
return JSONResponse(
status_code=401,
content={
"code": 401,
"message": exc.message,
},
)
@app.exception_handler(PermissionDeniedError)
async def permission_handler(
request: Request, exc: PermissionDeniedError
) -> JSONResponse:
return JSONResponse(
status_code=403,
content={
"code": 403,
"message": exc.message,
},
)
@app.exception_handler(FileTooLargeError)
async def file_too_large_handler(
request: Request, exc: FileTooLargeError
) -> JSONResponse:
return JSONResponse(
status_code=413,
content={
"code": 413,
"message": exc.message,
},
)
@app.exception_handler(InvalidFileTypeError)
async def invalid_file_type_handler(
request: Request, exc: InvalidFileTypeError
) -> JSONResponse:
return JSONResponse(
status_code=415,
content={
"code": 415,
"message": exc.message,
},
)
@app.exception_handler(RateLimitError)
async def rate_limit_handler(
request: Request, exc: RateLimitError
) -> JSONResponse:
return JSONResponse(
status_code=429,
content={
"code": 429,
"message": exc.message,
},
)
"""
Logging configuration using Loguru
"""
import sys
from pathlib import Path
from loguru import logger
from app.core.config import settings
def setup_logging():
"""Configure application logging"""
# Remove default handler
logger.remove()
# Add console handler
logger.add(
sys.stdout,
format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
level=settings.log_level,
colorize=True,
)
# Add file handler if log_file is configured
if settings.log_file:
log_path = Path(settings.log_file)
log_path.parent.mkdir(parents=True, exist_ok=True)
logger.add(
settings.log_file,
format="{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | {name}:{function}:{line} - {message}",
level=settings.log_level,
rotation="100 MB",
retention="30 days",
compression="zip",
)
# Set logger as global
logger.configure(
handlers=[
{
"sink": sys.stdout,
"level": settings.log_level,
"format": "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
}
]
)
return logger
def get_logger():
"""Get the configured logger"""
return logger
"""
Custom middleware for PLabel
"""
import time
from typing import Callable
from fastapi import Request, Response
from starlette.middleware.base import BaseHTTPMiddleware
from app.core.logging import get_logger
logger = get_logger()
class RequestLogMiddleware(BaseHTTPMiddleware):
"""Middleware to log all incoming requests and responses"""
async def dispatch(self, request: Request, call_next: Callable) -> Response:
request_id = request.headers.get("X-Request-ID", "-")
start_time = time.time()
method = request.method
url = str(request.url)
logger.info(
f"[{request_id}] --> {method} {url} | "
f"client={request.client.host if request.client else 'unknown'}"
)
try:
response = await call_next(request)
except Exception as exc:
elapsed = time.time() - start_time
logger.error(
f"[{request_id}] <-- {method} {url} | "
f"status=500 | elapsed={elapsed:.3f}s | error={exc}"
)
raise
elapsed = time.time() - start_time
logger.info(
f"[{request_id}] <-- {method} {url} | "
f"status={response.status_code} | elapsed={elapsed:.3f}s"
)
response.headers["X-Request-ID"] = request_id
response.headers["X-Response-Time"] = f"{elapsed:.3f}s"
return response
"""
Redis connection and caching utilities
"""
from typing import Optional
import redis.asyncio as aioredis
from redis import Redis
from redis.asyncio import Redis as AsyncRedis
from app.core.config import settings
# ===========================================
# Synchronous Redis Client
# ===========================================
def get_redis_client() -> Redis:
"""Get synchronous Redis client"""
return Redis(
host=settings.redis_host,
port=settings.redis_port,
db=settings.redis_db,
password=settings.redis_password or None,
decode_responses=True,
)
# ===========================================
# Async Redis Client
# ===========================================
_async_redis_client: Optional[AsyncRedis] = None
async def get_async_redis() -> AsyncRedis:
"""Get async Redis client (singleton pattern)"""
global _async_redis_client
if _async_redis_client is None:
_async_redis_client = await aioredis.from_url(
settings.redis_url,
encoding="utf-8",
decode_responses=True,
)
return _async_redis_client
async def close_async_redis():
"""Close async Redis connection"""
global _async_redis_client
if _async_redis_client is not None:
await _async_redis_client.close()
_async_redis_client = None
# ===========================================
# Cache Utilities
# ===========================================
class CacheService:
"""Simple caching service using Redis"""
def __init__(self, redis_client: AsyncRedis):
self.redis = redis_client
async def get(self, key: str) -> Optional[str]:
"""Get value from cache"""
return await self.redis.get(key)
async def set(
self,
key: str,
value: str,
expire: Optional[int] = None
) -> None:
"""Set value in cache with optional expiration (seconds)"""
await self.redis.set(key, value, ex=expire)
async def delete(self, key: str) -> None:
"""Delete key from cache"""
await self.redis.delete(key)
async def exists(self, key: str) -> bool:
"""Check if key exists"""
return await self.redis.exists(key) > 0
async def incr(self, key: str, amount: int = 1) -> int:
"""Increment counter"""
return await self.redis.incrby(key, amount)
async def decr(self, key: str, amount: int = 1) -> int:
"""Decrement counter"""
return await self.redis.decrby(key, amount)
async def get_cache() -> CacheService:
"""Get cache service instance"""
redis = await get_async_redis()
return CacheService(redis)
"""
Security utilities for authentication and authorization
"""
from datetime import datetime, timedelta
from typing import Optional
from jose import JWTError, jwt
from passlib.context import CryptContext
from app.core.config import settings
# Password hashing context
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
def verify_password(plain_password: str, hashed_password: str) -> bool:
"""Verify a plain password against a hashed password"""
return pwd_context.verify(plain_password, hashed_password)
def hash_password(password: str) -> str:
"""Hash a password using bcrypt"""
return pwd_context.hash(password)
def create_access_token(
data: dict,
expires_delta: Optional[timedelta] = None
) -> str:
"""
Create a JWT access token
Args:
data: Dictionary containing token payload (should include 'sub' for user_id)
expires_delta: Optional expiration time delta
Returns:
Encoded JWT token string
"""
to_encode = data.copy()
if expires_delta:
expire = datetime.utcnow() + expires_delta
else:
expire = datetime.utcnow() + timedelta(
minutes=settings.access_token_expire_minutes
)
to_encode.update({"exp": expire, "iat": datetime.utcnow()})
encoded_jwt = jwt.encode(
to_encode,
settings.jwt_secret_key,
algorithm=settings.jwt_algorithm
)
return encoded_jwt
def decode_token(token: str) -> Optional[dict]:
"""
Decode and verify a JWT token
Args:
token: JWT token string
Returns:
Decoded token payload or None if invalid
"""
try:
payload = jwt.decode(
token,
settings.jwt_secret_key,
algorithms=[settings.jwt_algorithm]
)
return payload
except JWTError:
return None
def create_refresh_token(
data: dict,
expires_delta: Optional[timedelta] = None
) -> str:
"""
Create a JWT refresh token with longer expiration
Args:
data: Dictionary containing token payload
expires_delta: Optional expiration time delta
Returns:
Encoded JWT refresh token string
"""
to_encode = data.copy()
if expires_delta:
expire = datetime.utcnow() + expires_delta
else:
expire = datetime.utcnow() + timedelta(
days=settings.refresh_token_expire_days
)
to_encode.update({
"exp": expire,
"iat": datetime.utcnow(),
"type": "refresh"
})
encoded_jwt = jwt.encode(
to_encode,
settings.jwt_secret_key,
algorithm=settings.jwt_algorithm
)
return encoded_jwt
"""
MinIO / S3-compatible object storage client wrapper
"""
import io
from pathlib import Path
from typing import Optional
from minio import Minio
from minio.datatypes import Object
from app.core.config import settings
from app.core.exceptions import StorageError
from app.core.logging import get_logger
logger = get_logger()
class StorageService:
"""MinIO storage service for file operations"""
def __init__(self):
self._client: Optional[Minio] = None
self._bucket = settings.minio_bucket
@property
def client(self) -> Minio:
"""Lazy-initialize MinIO client"""
if self._client is None:
self._client = Minio(
endpoint=settings.minio_endpoint,
access_key=settings.minio_access_key,
secret_key=settings.minio_secret_key,
secure=settings.minio_secure,
region=settings.minio_region,
)
return self._client
async def ensure_bucket_exists(self) -> None:
"""Ensure the default bucket exists, create if not"""
try:
if not self.client.bucket_exists(self._bucket):
logger.info(f"Creating bucket: {self._bucket}")
self.client.make_bucket(self._bucket)
logger.info(f"Bucket '{self._bucket}' created successfully")
except Exception as exc:
logger.error(f"Failed to create bucket '{self._bucket}': {exc}")
raise StorageError(f"Failed to create bucket: {exc}")
def upload_file(
self,
file_data: bytes,
object_name: str,
content_type: str = "application/octet-stream",
length: Optional[int] = None,
) -> str:
"""
Upload a file to MinIO
Args:
file_data: File content as bytes
object_name: Destination object name (path in bucket)
content_type: MIME type of the file
length: File size in bytes (defaults to len(file_data))
Returns:
The object name of the uploaded file
"""
try:
data = io.BytesIO(file_data)
file_length = length if length is not None else len(file_data)
self.client.put_object(
bucket_name=self._bucket,
object_name=object_name,
data=data,
length=file_length,
content_type=content_type,
)
logger.info(f"Uploaded file: {object_name} ({file_length} bytes)")
return object_name
except Exception as exc:
logger.error(f"Failed to upload file '{object_name}': {exc}")
raise StorageError(f"Failed to upload file: {exc}")
def upload_file_from_path(
self,
file_path: str,
object_name: str,
content_type: str = "application/octet-stream",
) -> str:
"""Upload a file from local filesystem to MinIO"""
try:
file_size = Path(file_path).stat().st_size
self.client.fput_object(
bucket_name=self._bucket,
object_name=object_name,
file_path=file_path,
content_type=content_type,
)
logger.info(
f"Uploaded file from path: {file_path} -> {object_name} ({file_size} bytes)"
)
return object_name
except Exception as exc:
logger.error(f"Failed to upload file from path '{file_path}': {exc}")
raise StorageError(f"Failed to upload file from path: {exc}")
def get_presigned_url(
self,
object_name: str,
expires: int = 3600,
) -> str:
"""
Generate a presigned URL for reading an object
Args:
object_name: Object name in bucket
expires: URL expiration time in seconds
Returns:
Presigned URL string
"""
try:
url = self.client.presigned_get_object(
bucket_name=self._bucket,
object_name=object_name,
expires=expires,
)
return url
except Exception as exc:
logger.error(f"Failed to generate presigned URL for '{object_name}': {exc}")
raise StorageError(f"Failed to generate presigned URL: {exc}")
def get_presigned_put_url(
self,
object_name: str,
expires: int = 3600,
content_type: str = "application/octet-stream",
) -> str:
"""
Generate a presigned URL for uploading an object directly
Args:
object_name: Destination object name
expires: URL expiration time in seconds
content_type: Expected content type
Returns:
Presigned PUT URL string
"""
try:
url = self.client.presigned_put_object(
bucket_name=self._bucket,
object_name=object_name,
expires=expires,
extra_query_params={"Content-Type": content_type},
)
return url
except Exception as exc:
logger.error(
f"Failed to generate presigned PUT URL for '{object_name}': {exc}"
)
raise StorageError(f"Failed to generate presigned PUT URL: {exc}")
def delete_file(self, object_name: str) -> None:
"""Delete an object from MinIO"""
try:
self.client.remove_object(
bucket_name=self._bucket,
object_name=object_name,
)
logger.info(f"Deleted file: {object_name}")
except Exception as exc:
logger.error(f"Failed to delete file '{object_name}': {exc}")
raise StorageError(f"Failed to delete file: {exc}")
def delete_files(self, object_names: list[str]) -> None:
"""Delete multiple objects from MinIO"""
try:
self.client.remove_objects(
bucket_name=self._bucket,
object_names=object_names,
)
logger.info(f"Deleted {len(object_names)} files")
except Exception as exc:
logger.error(f"Failed to delete multiple files: {exc}")
raise StorageError(f"Failed to delete multiple files: {exc}")
def list_files(
self,
prefix: str = "",
max_keys: int = 1000,
) -> list[Object]:
"""
List objects in the bucket with optional prefix filter
Args:
prefix: Filter objects by prefix
max_keys: Maximum number of objects to return
Returns:
List of Object items
"""
try:
objects = self.client.list_objects(
bucket_name=self._bucket,
prefix=prefix,
max_keys=max_keys,
)
return list(objects)
except Exception as exc:
logger.error(f"Failed to list files with prefix '{prefix}': {exc}")
raise StorageError(f"Failed to list files: {exc}")
def get_object(self, object_name: str) -> io.BytesIO:
"""Get an object's content as BytesIO"""
try:
response = self.client.get_object(
bucket_name=self._bucket,
object_name=object_name,
)
data = io.BytesIO(response.read())
response.close()
response.release_conn()
return data
except Exception as exc:
logger.error(f"Failed to get object '{object_name}': {exc}")
raise StorageError(f"Failed to get object: {exc}")
def stat_object(self, object_name: str) -> dict:
"""Get object metadata/statistics"""
try:
stat = self.client.stat_object(
bucket_name=self._bucket,
object_name=object_name,
)
return {
"size": stat.size,
"etag": stat.etag,
"content_type": stat.content_type,
"last_modified": stat.last_modified,
}
except Exception as exc:
logger.error(f"Failed to stat object '{object_name}': {exc}")
raise StorageError(f"Failed to stat object: {exc}")
def copy_object(
self,
source_object: str,
dest_object: str,
) -> str:
"""Copy an object within the same bucket"""
try:
self.client.copy_object(
bucket_name=self._bucket,
object_name=dest_object,
source=self.client.fcopy_object(
bucket_name=self._bucket,
object_name=source_object,
),
)
logger.info(f"Copied: {source_object} -> {dest_object}")
return dest_object
except Exception as exc:
logger.error(f"Failed to copy '{source_object}' to '{dest_object}': {exc}")
raise StorageError(f"Failed to copy object: {exc}")
storage_service = StorageService()
"""
PLabel Backend - FastAPI Application Entry Point
"""
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from app.core.config import settings
from app.core.database import async_engine, close_db
from app.core.exceptions import register_exception_handlers
from app.core.logging import setup_logging
from app.core.middleware import RequestLogMiddleware
from app.core.redis import close_async_redis
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Application lifespan events"""
# Startup
setup_logging()
yield
# Shutdown
await close_async_redis()
await close_db()
# Create FastAPI application
app = FastAPI(
title=settings.app_name,
description="AI Training Data Annotation Platform Backend",
version="1.0.0",
docs_url="/docs",
redoc_url="/redoc",
openapi_url="/openapi.json",
lifespan=lifespan,
)
# Configure CORS
app.add_middleware(
CORSMiddleware,
allow_origins=settings.cors_origins_list,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Add request logging middleware
app.add_middleware(RequestLogMiddleware)
# Register exception handlers
register_exception_handlers(app)
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
"""Global exception handler for unhandled exceptions"""
return JSONResponse(
status_code=500,
content={
"code": 500,
"message": "Internal server error",
"detail": str(exc) if settings.app_debug else "An error occurred"
}
)
@app.get("/")
async def root():
"""Root endpoint"""
return {
"name": settings.app_name,
"version": "1.0.0",
"status": "running"
}
@app.get("/health")
async def health_check():
"""Health check endpoint"""
return {"status": "healthy"}
# Import and include API routers
from app.api.v1 import api_router
app.include_router(api_router, prefix="/api/v1")
if __name__ == "__main__":
import uvicorn
uvicorn.run(
"app.main:app",
host=settings.host,
port=settings.port,
reload=settings.app_debug,
)
"""
PLabel Models Package
"""
from app.core.database import Base
from app.models.user import User
from app.models.project import Project
from app.models.project_member import ProjectMember
from app.models.label_set import LabelSet
from app.models.label import Label
from app.models.task import Task
from app.models.data_item import DataItem
from app.models.annotation import Annotation
from app.models.audio_annotation import AudioAnnotation
from app.models.text_entity import TextEntity
from app.models.text_relation import TextRelation
from app.models.review_session import ReviewSession, ReviewComment
from app.models.ai_model import AIModel
from app.models.auto_label_task import AutoLabelTask
__all__ = [
"Base",
"User",
"Project",
"ProjectMember",
"LabelSet",
"Label",
"Task",
"DataItem",
"Annotation",
"AudioAnnotation",
"TextEntity",
"TextRelation",
"ReviewSession",
"ReviewComment",
"AIModel",
"AutoLabelTask",
]
"""
AIModel model
"""
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import Integer, String, Text, Float
from sqlalchemy import JSON
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column
from app.core.database import Base
class AIModel(Base):
"""AIModel model for AI model configurations"""
__tablename__ = "ai_models"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
name: Mapped[str] = mapped_column(String(200), nullable=False)
type: Mapped[str] = mapped_column(String(30), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
framework: Mapped[str | None] = mapped_column(String(50), nullable=True)
model_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
config: Mapped[dict] = mapped_column(JSON, default=dict)
accuracy: Mapped[float | None] = mapped_column(Float, nullable=True)
speed: Mapped[int | None] = mapped_column(Integer, nullable=True)
enabled: Mapped[bool] = mapped_column(default=True)
"""
Annotation model (image/video annotations)
"""
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import Boolean, ForeignKey, Integer, String
from sqlalchemy import JSON
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.data_item import DataItem
from app.models.user import User
from app.models.label import Label
class Annotation(Base):
"""Annotation model for image/video bounding boxes, polygons, keypoints etc."""
__tablename__ = "annotations"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
data_item_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("data_items.id", ondelete="CASCADE"), nullable=False, index=True
)
user_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True)
label_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("labels.id"), nullable=True)
annotation_type: Mapped[str] = mapped_column(String(30), nullable=False)
label_name: Mapped[str] = mapped_column(String(100), nullable=False)
data: Mapped[dict] = mapped_column(JSON, nullable=False)
visible: Mapped[bool] = mapped_column(Boolean, default=True)
occluded: Mapped[bool] = mapped_column(Boolean, default=False)
ignored: Mapped[bool] = mapped_column(Boolean, default=False)
frame_index: Mapped[int] = mapped_column(Integer, default=0, index=True)
data_item: Mapped["DataItem"] = relationship("DataItem", back_populates="annotations")
"""
AudioAnnotation model
"""
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import Float, ForeignKey, String
from sqlalchemy import JSON
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.data_item import DataItem
from app.models.user import User
class AudioAnnotation(Base):
"""AudioAnnotation model for speech transcription with timestamps"""
__tablename__ = "audio_annotations"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
data_item_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("data_items.id", ondelete="CASCADE"), nullable=False, index=True
)
user_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True)
start_time: Mapped[float] = mapped_column(Float, nullable=False)
end_time: Mapped[float] = mapped_column(Float, nullable=False)
text: Mapped[str | None] = mapped_column(nullable=True)
speaker: Mapped[str | None] = mapped_column(String(100), nullable=True)
labels: Mapped[list] = mapped_column(JSON, default=list)
confidence: Mapped[float | None] = mapped_column(Float, nullable=True)
data_item: Mapped["DataItem"] = relationship("DataItem")
"""
AutoLabelTask model
"""
import uuid
from typing import TYPE_CHECKING, Optional
from sqlalchemy import DateTime, ForeignKey, Integer, String, Text
from sqlalchemy import JSON
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.task import Task
from app.models.user import User
from app.models.ai_model import AIModel
class AutoLabelTask(Base):
"""AutoLabelTask model for AI-assisted auto-labeling jobs"""
__tablename__ = "auto_label_tasks"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
task_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("tasks.id", ondelete="CASCADE"), nullable=False, index=True
)
user_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False)
model_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("ai_models.id"), nullable=False, index=True
)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True)
progress: Mapped[int] = mapped_column(Integer, default=0)
config: Mapped[dict] = mapped_column(JSON, default=dict)
result: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
data_item_ids: Mapped[list] = mapped_column(JSON, default=list)
started_at: Mapped[Optional[uuid.UUID]] = mapped_column(nullable=True)
completed_at: Mapped[Optional[uuid.UUID]] = mapped_column(nullable=True)
task: Mapped["Task"] = relationship("Task")
ai_model: Mapped["AIModel"] = relationship("AIModel")
"""
Base model classes and mixins
Note: The actual ORM Base class is defined in app.core.database (Base).
These mixins are composed with Base in each model.
"""
import uuid
from datetime import datetime
from sqlalchemy import DateTime, func
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, declared_attr
class TimestampMixin:
"""Mixin that adds created_at and updated_at timestamp columns"""
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
nullable=False,
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now(),
nullable=False,
)
"""
DataItem model
"""
from typing import TYPE_CHECKING, Optional
from sqlalchemy import BigInteger, Float, ForeignKey, Integer, String
from sqlalchemy import JSON
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.task import Task
from app.models.annotation import Annotation
class DataItem(Base):
"""DataItem model for uploaded files/data samples"""
__tablename__ = "data_items"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
task_id: Mapped[int] = mapped_column(
Integer, ForeignKey("tasks.id", ondelete="CASCADE"), nullable=False, index=True
)
name: Mapped[str] = mapped_column(String(500), nullable=False)
type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
file_path: Mapped[str] = mapped_column(String(1000), nullable=False)
thumbnail_path: Mapped[Optional[str]] = mapped_column(String(1000), nullable=True)
size: Mapped[int] = mapped_column(BigInteger, nullable=False)
mime_type: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
width: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
height: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
duration: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
pages: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
meta: Mapped[dict] = mapped_column(JSON, default=dict)
annotation_status: Mapped[str] = mapped_column(String(20), default="none", index=True)
task: Mapped["Task"] = relationship("Task", back_populates="data_items")
annotations: Mapped[list["Annotation"]] = relationship("Annotation", back_populates="data_item", cascade="all, delete-orphan")
"""
Label model
"""
from typing import TYPE_CHECKING, Optional
from sqlalchemy import ForeignKey, Integer, String
from sqlalchemy import JSON
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.label_set import LabelSet
class Label(Base):
"""Label model for annotation categories"""
__tablename__ = "labels"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
label_set_id: Mapped[int] = mapped_column(
Integer, ForeignKey("label_sets.id", ondelete="CASCADE"), nullable=False, index=True
)
name: Mapped[str] = mapped_column(String(100), nullable=False)
color: Mapped[str] = mapped_column(String(7), nullable=False)
type: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
shortcut: Mapped[Optional[str]] = mapped_column(String(10), nullable=True)
parent_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("labels.id"), nullable=True)
attributes: Mapped[dict] = mapped_column(JSON, default=list)
label_set: Mapped["LabelSet"] = relationship("LabelSet", back_populates="labels")
"""
LabelSet model
"""
from typing import TYPE_CHECKING, Optional
from sqlalchemy import ForeignKey, Integer, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.project import Project
from app.models.label import Label
class LabelSet(Base):
"""LabelSet model for grouping labels within a project"""
__tablename__ = "label_sets"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
project_id: Mapped[Optional[int]] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=True, index=True
)
name: Mapped[str] = mapped_column(String(200), nullable=False)
project: Mapped[Optional["Project"]] = relationship("Project", back_populates="label_set")
labels: Mapped[list["Label"]] = relationship("Label", back_populates="label_set", cascade="all, delete-orphan")
"""
Project model
"""
from typing import TYPE_CHECKING, Optional
from sqlalchemy import ForeignKey, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.user import User
from app.models.label_set import LabelSet
from app.models.task import Task
from app.models.project_member import ProjectMember
class Project(Base):
"""Project model for organizing annotation tasks"""
__tablename__ = "projects"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
name: Mapped[str] = mapped_column(String(200), nullable=False)
description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="active", index=True)
owner_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
)
label_set_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
owner: Mapped["User"] = relationship("User", back_populates="owned_projects")
label_set: Mapped[Optional["LabelSet"]] = relationship("LabelSet", foreign_keys=[label_set_id], post_update=True)
tasks: Mapped[list["Task"]] = relationship("Task", back_populates="project", cascade="all, delete-orphan")
members: Mapped[list["ProjectMember"]] = relationship("ProjectMember", back_populates="project", cascade="all, delete-orphan")
"""
ProjectMember model
"""
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, Integer, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.project import Project
from app.models.user import User
class ProjectMember(Base):
"""ProjectMember model for project access control"""
__tablename__ = "project_members"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
project_id: Mapped[int] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True
)
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
)
role: Mapped[str] = mapped_column(String(20), nullable=False, default="member")
project: Mapped["Project"] = relationship("Project", back_populates="members")
user: Mapped["User"] = relationship("User", back_populates="project_memberships")
"""
ReviewSession and ReviewComment models
"""
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, String, Text
from sqlalchemy import Boolean
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.task import Task
from app.models.data_item import DataItem
from app.models.user import User
class ReviewSession(Base):
"""ReviewSession model for annotation quality review"""
__tablename__ = "review_sessions"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
task_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("tasks.id", ondelete="CASCADE"), nullable=False, index=True
)
data_item_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("data_items.id", ondelete="CASCADE"), nullable=False, index=True
)
reviewer_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True
)
annotator_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True)
task: Mapped["Task"] = relationship("Task")
data_item: Mapped["DataItem"] = relationship("DataItem")
reviewer: Mapped["User"] = relationship("User", foreign_keys=[reviewer_id])
annotator: Mapped["User | None"] = relationship("User", foreign_keys=[annotator_id])
comments: Mapped[list["ReviewComment"]] = relationship("ReviewComment", back_populates="session", cascade="all, delete-orphan")
class ReviewComment(Base):
"""ReviewComment model for review feedback"""
__tablename__ = "review_comments"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
session_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("review_sessions.id", ondelete="CASCADE"), nullable=False, index=True
)
reviewer_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False)
annotation_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), nullable=True)
content: Mapped[str] = mapped_column(Text, nullable=False)
resolved: Mapped[bool] = mapped_column(Boolean, default=False)
session: Mapped["ReviewSession"] = relationship("ReviewSession", back_populates="comments")
"""
Task model
"""
from typing import TYPE_CHECKING, Optional
from sqlalchemy import ForeignKey, Integer, String, Text
from sqlalchemy import JSON
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.project import Project
from app.models.user import User
from app.models.ai_model import AIModel
from app.models.data_item import DataItem
class Task(Base):
"""Task model for annotation work units"""
__tablename__ = "tasks"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
project_id: Mapped[int] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True
)
name: Mapped[str] = mapped_column(String(200), nullable=False)
description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
data_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
annotation_types: Mapped[list] = mapped_column(JSON, default=list)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True)
assignee_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("users.id"), nullable=True, index=True)
reviewer_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("users.id"), nullable=True)
ai_model_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("ai_models.id"), nullable=True)
progress: Mapped[int] = mapped_column(Integer, default=0)
total_count: Mapped[int] = mapped_column(Integer, default=0)
completed_count: Mapped[int] = mapped_column(Integer, default=0)
reviewed_count: Mapped[int] = mapped_column(Integer, default=0)
rejected_count: Mapped[int] = mapped_column(Integer, default=0)
project: Mapped["Project"] = relationship("Project", back_populates="tasks")
assignee: Mapped[Optional["User"]] = relationship("User", foreign_keys=[assignee_id])
reviewer: Mapped[Optional["User"]] = relationship("User", foreign_keys=[reviewer_id])
ai_model: Mapped[Optional["AIModel"]] = relationship("AIModel")
data_items: Mapped[list["DataItem"]] = relationship("DataItem", back_populates="task", cascade="all, delete-orphan")
"""
TextEntity model
"""
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, Integer, String
from sqlalchemy import JSON
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.data_item import DataItem
from app.models.user import User
class TextEntity(Base):
"""TextEntity model for named entity recognition"""
__tablename__ = "text_entities"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
data_item_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("data_items.id", ondelete="CASCADE"), nullable=False, index=True
)
user_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True)
label_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("labels.id"), nullable=True)
label_name: Mapped[str] = mapped_column(String(100), nullable=False)
start_offset: Mapped[int] = mapped_column(Integer, nullable=False)
end_offset: Mapped[int] = mapped_column(Integer, nullable=False)
text: Mapped[str] = mapped_column(String(1000), nullable=False)
color: Mapped[str | None] = mapped_column(String(7), nullable=True)
attributes: Mapped[dict] = mapped_column(JSON, default=dict)
data_item: Mapped["DataItem"] = relationship("DataItem")
"""
TextRelation model
"""
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, String
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column
from app.core.database import Base
if TYPE_CHECKING:
from app.models.data_item import DataItem
from app.models.user import User
class TextRelation(Base):
"""TextRelation model for entity relationship annotation"""
__tablename__ = "text_relations"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
data_item_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("data_items.id", ondelete="CASCADE"), nullable=False, index=True
)
user_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True)
from_entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("text_entities.id"), nullable=False)
to_entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("text_entities.id"), nullable=False)
label: Mapped[str] = mapped_column(String(100), nullable=False)
relation_type: Mapped[str] = mapped_column(String(20), default="undirected")
"""
User model
"""
import uuid
from typing import TYPE_CHECKING, Optional
from sqlalchemy import Boolean, String
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.core.database import Base
if TYPE_CHECKING:
from app.models.project import Project
from app.models.project_member import ProjectMember
class User(Base):
"""User model for authentication and authorization"""
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
username: Mapped[str] = mapped_column(String(50), unique=True, nullable=False, index=True)
email: Mapped[str] = mapped_column(String(255), unique=True, nullable=False, index=True)
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
avatar_url: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
role: Mapped[str] = mapped_column(String(20), nullable=False, default="user", index=True)
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
owned_projects: Mapped[list["Project"]] = relationship(
"Project", back_populates="owner", cascade="all, delete-orphan"
)
project_memberships: Mapped[list["ProjectMember"]] = relationship(
"ProjectMember", back_populates="user", cascade="all, delete-orphan"
)
def __repr__(self) -> str:
return f"<User(id={self.id}, username={self.username}, role={self.role})>"
"""
PLabel Schemas Package
"""
from app.schemas.base import BaseSchema, ErrorResponseSchema, PaginationSchema, ResponseSchema
from app.schemas.auth import (
ChangePasswordRequest,
LoginRequest,
LoginResponse,
RegisterRequest,
TokenPayload,
TokenResponse,
UserResponse,
)
from app.schemas.user import (
UserCreate,
UserInDB,
UserStatsResponse,
UserUpdate,
)
from app.schemas.project import (
ProjectCreate,
ProjectMemberAdd,
ProjectMemberResponse,
ProjectStatistics,
ProjectUpdate,
)
from app.schemas.label import (
LabelCreate,
LabelSchema,
LabelSetCreate,
LabelSetUpdate,
LabelUpdate,
)
from app.schemas.task import (
DataType,
TaskCreate,
TaskProgressResponse,
TaskStatus,
TaskUpdate,
)
from app.schemas.data_item import (
AnnotationStatus,
DataItemMetadata,
DataItemUpdate,
)
from app.schemas.annotation import (
AnnotationCreate,
AnnotationType,
AudioAnnotationCreate,
BatchAnnotationRequest,
BoundingBoxData,
BrushData,
KeypointData,
PolygonData,
SkeletonData,
TextEntityCreate,
TextRelationCreate,
)
from app.schemas.review import (
BatchReviewRequest,
ReviewAction,
ReviewCommentCreate,
ReviewSessionCreate,
ReviewStatus,
)
from app.schemas.ai_model import (
AIModelCreate,
AIModelTestRequest,
AIModelType,
AIModelUpdate,
)
from app.schemas.autolabel import (
AutoLabelProgressResponse,
AutoLabelStatus,
AutoLabelTaskCreate,
)
from app.schemas.export import (
ExportFormat,
ExportOptions,
ExportStatus,
ExportTaskCreate,
)
from app.schemas.statistics import (
DashboardStats,
TaskStatistics,
WeeklyReportStats,
)
from app.schemas.upload import (
MultipartUploadInit,
MultipartUploadPart,
PresignedUrlRequest,
PresignedUrlResponse,
)
__all__ = [
"BaseSchema",
"PaginationSchema",
"ResponseSchema",
"ErrorResponseSchema",
"LoginRequest",
"RegisterRequest",
"LoginResponse",
"TokenResponse",
"TokenPayload",
"UserResponse",
"UserCreate",
"UserUpdate",
"UserInDB",
"UserStatsResponse",
"ProjectCreate",
"ProjectUpdate",
"ProjectMemberAdd",
"ProjectMemberResponse",
"ProjectStatistics",
"LabelSetCreate",
"LabelSetUpdate",
"LabelCreate",
"LabelUpdate",
"LabelSchema",
"TaskStatus",
"DataType",
"TaskCreate",
"TaskUpdate",
"TaskProgressResponse",
"AnnotationStatus",
"DataItemUpdate",
"DataItemMetadata",
"AnnotationType",
"AnnotationCreate",
"BatchAnnotationRequest",
"BoundingBoxData",
"PolygonData",
"BrushData",
"KeypointData",
"SkeletonData",
"AudioAnnotationCreate",
"TextEntityCreate",
"TextRelationCreate",
"ReviewStatus",
"ReviewSessionCreate",
"ReviewAction",
"BatchReviewRequest",
"ReviewCommentCreate",
"AIModelType",
"AIModelCreate",
"AIModelUpdate",
"AIModelTestRequest",
"AutoLabelStatus",
"AutoLabelTaskCreate",
"AutoLabelProgressResponse",
"ExportFormat",
"ExportStatus",
"ExportTaskCreate",
"ExportOptions",
"DashboardStats",
"TaskStatistics",
"WeeklyReportStats",
"PresignedUrlRequest",
"PresignedUrlResponse",
"MultipartUploadInit",
"MultipartUploadPart",
]
"""
AI Model schemas
"""
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class AIModelType(str, Enum):
OBJECT_DETECTION = "object_detection"
SEGMENTATION = "segmentation"
KEYPOINT = "keypoint"
CLASSIFICATION = "classification"
ASR = "asr"
NER = "ner"
OCR = "ocr"
class AIModelCreate(BaseModel):
name: str = Field(min_length=1, max_length=200)
type: AIModelType
description: str | None = None
framework: str | None = None
model_path: str | None = None
config: dict = {}
accuracy: float | None = Field(default=None, le=1.0, ge=0.0)
speed: int | None = None
class AIModelUpdate(BaseModel):
name: str | None = None
description: str | None = None
config: dict | None = None
enabled: bool | None = None
class AIModelTestRequest(BaseModel):
input_data: dict
"""
Annotation schemas
"""
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class AnnotationType(str, Enum):
RECTANGLE = "rectangle"
POLYGON = "polygon"
BRUSH = "brush"
KEYPOINT = "keypoint"
SKELETON = "skeleton"
TEXT = "text"
class BoundingBoxData(BaseModel):
x: float
y: float
width: float
height: float
rotation: float = 0
class PolygonData(BaseModel):
points: list[dict]
closed: bool = True
class BrushData(BaseModel):
mask: str
width: int
height: int
class KeypointData(BaseModel):
points: list[dict]
class SkeletonData(BaseModel):
points: list[dict]
connections: list[list[int]]
template_id: str | None = None
class AnnotationCreate(BaseModel):
data_item_id: str
label_id: str | None = None
type: AnnotationType
data: dict
frame_index: int = 0
visible: bool = True
occluded: bool = False
ignored: bool = False
class BatchAnnotationRequest(BaseModel):
data_item_id: str
frame_index: int = 0
annotations: list[AnnotationCreate]
class AudioAnnotationCreate(BaseModel):
data_item_id: str
start_time: float = Field(ge=0)
end_time: float = Field(gt=0)
text: str | None = None
speaker: str | None = None
labels: list[str] = []
class TextEntityCreate(BaseModel):
data_item_id: str
label_id: str | None = None
start_offset: int = Field(ge=0)
end_offset: int = Field(gt=0)
text: str
attributes: dict = {}
class TextRelationCreate(BaseModel):
data_item_id: str
from_entity_id: str
to_entity_id: str
label: str
relation_type: str = "undirected"
"""
Auth schemas
"""
from pydantic import BaseModel, EmailStr, Field
from app.schemas.base import BaseSchema
class LoginRequest(BaseModel):
identifier: str = Field(..., description="邮箱或用户名")
password: str
class RegisterRequest(BaseModel):
username: str = Field(min_length=2, max_length=50)
email: EmailStr
password: str = Field(min_length=8, max_length=128)
class TokenResponse(BaseSchema):
access_token: str
token_type: str = "bearer"
expires_in: int
class UserResponse(BaseSchema):
id: str
username: str
email: str
role: str
avatar_url: str | None = None
is_active: bool = True
class LoginResponse(BaseSchema):
access_token: str
token_type: str = "bearer"
expires_in: int
user: UserResponse
class ChangePasswordRequest(BaseModel):
old_password: str
new_password: str = Field(min_length=8, max_length=128)
class TokenPayload(BaseModel):
sub: str
email: str
role: str
exp: int
iat: int
"""
AutoLabel schemas
"""
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class AutoLabelStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
class AutoLabelTaskCreate(BaseModel):
task_id: str
data_item_ids: list[str] = []
model_id: str
config: dict = {}
class AutoLabelProgressResponse(BaseSchema):
status: str
progress: int = 0
processed_count: int = 0
total_count: int = 0
started_at: datetime | None = None
completed_at: datetime | None = None
error: str | None = None
"""
Base Pydantic schemas
"""
from datetime import datetime
from typing import Any, Generic, TypeVar
from pydantic import BaseModel, ConfigDict, Field
class BaseSchema(BaseModel):
"""Base schema with common configuration"""
model_config = ConfigDict(from_attributes=True)
class PaginationSchema(BaseSchema):
"""Pagination response base"""
items: list[Any]
total: int = Field(ge=0, description="Total number of records")
page: int = Field(ge=1, description="Current page number")
page_size: int = Field(ge=1, le=100, description="Number of records per page")
total_pages: int = Field(ge=0, description="Total number of pages")
class ResponseSchema(BaseSchema):
"""Standard API response wrapper"""
code: int = 200
message: str = "success"
data: Any = None
class ErrorResponseSchema(BaseSchema):
"""Standard error response"""
code: int = 400
message: str
errors: list[dict] | None = None
details: Any = None
"""
DataItem schemas
"""
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class AnnotationStatus(str, Enum):
NONE = "none"
PARTIAL = "partial"
COMPLETED = "completed"
REJECTED = "rejected"
class DataItemUpdate(BaseModel):
name: str | None = None
metadata: dict | None = Field(default=None, validation_alias="meta")
class DataItemMetadata(BaseSchema):
width: int | None = None
height: int | None = None
duration: float | None = None
pages: int | None = None
format: str | None = None
fps: float | None = None
frame_count: int | None = None
char_count: int | None = None
word_count: int | None = None
"""
Export schemas
"""
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class ExportFormat(str, Enum):
COCO_JSON = "coco_json"
VOC_XML = "voc_xml"
YOLO_TXT = "yolo_txt"
MASK_PNG = "mask_png"
COCO_KEYPOINT = "coco_keypoint"
SRT = "srt"
VTT = "vtt"
TEXTGRID = "textgrid"
JSON = "json"
CSV = "csv"
CONLL = "conll"
BIO = "bio"
class ExportStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
class ExportTaskCreate(BaseModel):
task_id: str
format: ExportFormat
options: dict = {}
class ExportOptions(BaseSchema):
include_images: bool = True
split_ratio: float = 0.8
annotation_types: list[str] = []
"""
Label schemas
"""
from datetime import datetime
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class LabelCreate(BaseModel):
name: str = Field(min_length=1, max_length=100)
color: str = Field(pattern=r"^#[0-9A-Fa-f]{6}$")
type: str = Field(description="rectangle/polygon/brush/keypoint/skeleton/text/classification/relation")
shortcut: str | None = Field(default=None, max_length=10)
parent_id: str | None = None
attributes: list[dict] = []
class LabelUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=100)
color: str | None = Field(default=None, pattern=r"^#[0-9A-Fa-f]{6}$")
shortcut: str | None = Field(default=None, max_length=10)
attributes: list[dict] | None = None
class LabelSchema(BaseSchema):
id: str
label_set_id: str
name: str
color: str
type: str
shortcut: str | None = None
parent_id: str | None = None
attributes: list[dict] = []
created_at: datetime
class LabelSetCreate(BaseModel):
name: str = Field(min_length=1, max_length=200)
class LabelSetUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=200)
labels: list[LabelCreate] = []
"""
Project schemas
"""
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class ProjectCreate(BaseModel):
name: str = Field(min_length=1, max_length=200)
description: str | None = None
class ProjectUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=200)
description: str | None = None
status: str | None = None
class ProjectMemberAdd(BaseModel):
user_id: str
role: str = "member"
class ProjectMemberResponse(BaseSchema):
id: str
project_id: str
user_id: str
role: str
joined_at: datetime
class ProjectStatistics(BaseSchema):
total_tasks: int = 0
completed_tasks: int = 0
pending_tasks: int = 0
in_progress_tasks: int = 0
reviewed_tasks: int = 0
total_annotations: int = 0
total_members: int = 0
progress_percentage: float = 0.0
"""
Review schemas
"""
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class ReviewStatus(str, Enum):
PENDING = "pending"
APPROVED = "approved"
REJECTED = "rejected"
class ReviewSessionCreate(BaseModel):
task_id: str
data_item_id: str
class ReviewAction(BaseModel):
comments: str | None = None
reason: str | None = None
annotation_ids: list[str] | None = None
class BatchReviewRequest(BaseModel):
task_id: str
data_item_ids: list[str]
action: str = Field(pattern="^(approve|reject)$")
reason: str | None = None
class ReviewCommentCreate(BaseModel):
content: str = Field(min_length=1)
annotation_id: str | None = None
"""
Statistics schemas
"""
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class DashboardStats(BaseSchema):
total_projects: int = 0
total_tasks: int = 0
total_annotations: int = 0
pending_tasks: int = 0
in_progress_tasks: int = 0
completed_tasks: int = 0
reviewed_tasks: int = 0
class TaskStatistics(BaseSchema):
by_status: dict[str, int] = {}
by_data_type: dict[str, int] = {}
total: int = 0
class WeeklyReportStats(BaseSchema):
week: str
annotation_count: int = 0
task_completed_count: int = 0
review_approved_count: int = 0
review_rejected_count: int = 0
daily_counts: list[dict] = []
"""
Task schemas
"""
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field
from app.schemas.base import BaseSchema
class TaskStatus(str, Enum):
PENDING = "pending"
IN_PROGRESS = "in_progress"
SUBMITTED = "submitted"
REJECTED = "rejected"
REVIEWED = "reviewed"
EXPORTED = "exported"
PAUSED = "paused"
class DataType(str, Enum):
IMAGE = "image"
VIDEO = "video"
AUDIO = "audio"
TEXT = "text"
class TaskCreate(BaseModel):
project_id: str
name: str = Field(min_length=1, max_length=200)
description: str | None = None
data_type: DataType
annotation_types: list[str] = []
assignee_id: str | None = None
ai_model_id: str | None = None
class TaskUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=200)
description: str | None = None
status: TaskStatus | None = None
assignee_id: str | None = None
reviewer_id: str | None = None
ai_model_id: str | None = None
class TaskProgressResponse(BaseSchema):
progress: int = 0
total_count: int = 0
completed_count: int = 0
reviewed_count: int = 0
rejected_count: int = 0
"""
Upload schemas
"""
from pydantic import BaseModel
from app.schemas.base import BaseSchema
class PresignedUrlRequest(BaseModel):
file_name: str
content_type: str
class PresignedUrlResponse(BaseSchema):
upload_url: str
file_key: str
expires_in: int
class MultipartUploadInit(BaseModel):
file_name: str
file_size: int
content_type: str
class MultipartUploadPart(BaseModel):
part_number: int
etag: str
"""
User schemas
"""
from datetime import datetime
from typing import Any
from pydantic import BaseModel, EmailStr, Field
from app.schemas.base import BaseSchema
class UserBase(BaseSchema):
username: str = Field(min_length=2, max_length=50)
email: EmailStr
role: str = "user"
avatar_url: str | None = None
is_active: bool = True
class UserCreate(BaseModel):
username: str = Field(min_length=2, max_length=50)
email: EmailStr
password: str = Field(min_length=8, max_length=128)
role: str = "user"
class UserUpdate(BaseModel):
username: str | None = Field(default=None, min_length=2, max_length=50)
email: EmailStr | None = None
avatar_url: str | None = None
role: str | None = None
is_active: bool | None = None
class UserInDB(BaseSchema):
id: str
username: str
email: str
password_hash: str
role: str
avatar_url: str | None = None
is_active: bool
created_at: datetime
updated_at: datetime
class UserStatsResponse(BaseSchema):
total_annotations: int = 0
completed_tasks: int = 0
reviewed_tasks: int = 0
weekly_annotation_count: int = 0
total_working_hours: float = 0.0
class UserResponse(BaseSchema):
"""Response schema for user data"""
id: str
username: str
email: str
role: str
avatar_url: str | None = None
is_active: bool = True
"""
Authentication service for user login, registration, and token management
"""
from datetime import timedelta
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.exceptions import AuthenticationError, ConflictError, ValidationError
from app.core.security import create_access_token, hash_password, verify_password
from app.models.user import User
from app.schemas.auth import LoginRequest, RegisterRequest
class AuthService:
"""Service for handling authentication operations"""
def __init__(self, db: AsyncSession):
self.db = db
async def authenticate_user(self, identifier: str, password: str) -> User:
"""
Authenticate user by email or username and password
Args:
identifier: Email address or username
password: User password
"""
from sqlalchemy import or_
# Try to find user by email or username
result = await self.db.execute(
select(User).where(
or_(
User.email == identifier,
User.username == identifier
)
)
)
user = result.scalar_one_or_none()
if not user:
raise AuthenticationError("Invalid email/username or password")
if not verify_password(password, user.password_hash):
raise AuthenticationError("Invalid email/username or password")
if not user.is_active:
raise AuthenticationError("User account is disabled")
return user
async def login(self, login_data: LoginRequest) -> tuple[User, str]:
"""
Login user and return user and access token
Returns:
Tuple of (User, access_token)
"""
user = await self.authenticate_user(login_data.identifier, login_data.password)
access_token = create_access_token(
data={
"sub": str(user.id),
"email": user.email,
"role": user.role,
}
)
return user, access_token
async def register(self, register_data: RegisterRequest) -> User:
"""Register a new user"""
existing_email = await self.db.execute(
select(User).where(User.email == register_data.email)
)
if existing_email.scalar_one_or_none():
raise ConflictError("Email already registered")
existing_username = await self.db.execute(
select(User).where(User.username == register_data.username)
)
if existing_username.scalar_one_or_none():
raise ConflictError("Username already taken")
if len(register_data.password) < 8:
raise ValidationError("Password must be at least 8 characters long")
password_hash = hash_password(register_data.password)
new_user = User(
username=register_data.username,
email=register_data.email,
password_hash=password_hash,
role="user",
is_active=True,
)
self.db.add(new_user)
await self.db.commit()
await self.db.refresh(new_user)
return new_user
async def get_user_by_id(self, user_id: int) -> User | None:
"""Get user by ID"""
result = await self.db.execute(
select(User).where(User.id == user_id)
)
return result.scalar_one_or_none()
async def get_user_by_email(self, email: str) -> User | None:
"""Get user by email"""
result = await self.db.execute(
select(User).where(User.email == email)
)
return result.scalar_one_or_none()
"""
User service for user management operations
"""
from typing import Sequence
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.exceptions import ConflictError, NotFoundError, PermissionDeniedError
from app.core.security import hash_password
from app.models.user import User
from app.schemas.user import UserCreate, UserUpdate, UserStatsResponse
class UserService:
"""Service for handling user management operations"""
def __init__(self, db: AsyncSession):
self.db = db
async def get_by_id(self, user_id: int) -> User | None:
"""Get user by ID"""
result = await self.db.execute(
select(User).where(User.id == user_id)
)
return result.scalar_one_or_none()
async def get_by_email(self, email: str) -> User | None:
"""Get user by email"""
result = await self.db.execute(
select(User).where(User.email == email)
)
return result.scalar_one_or_none()
async def get_by_username(self, username: str) -> User | None:
"""Get user by username"""
result = await self.db.execute(
select(User).where(User.username == username)
)
return result.scalar_one_or_none()
async def get_users(
self,
skip: int = 0,
limit: int = 20,
role: str | None = None,
) -> tuple[Sequence[User], int]:
"""Get paginated list of users"""
query = select(User)
count_query = select(func.count()).select_from(User)
if role:
query = query.where(User.role == role)
count_query = count_query.where(User.role == role)
total_result = await self.db.execute(count_query)
total = total_result.scalar() or 0
query = query.offset(skip).limit(limit).order_by(User.id)
result = await self.db.execute(query)
users = result.scalars().all()
return users, total
async def create_user(self, user_data: UserCreate) -> User:
"""Create a new user"""
existing_email = await self.get_by_email(user_data.email)
if existing_email:
raise ConflictError("Email already registered")
existing_username = await self.get_by_username(user_data.username)
if existing_username:
raise ConflictError("Username already taken")
password_hash = hash_password(user_data.password)
new_user = User(
username=user_data.username,
email=user_data.email,
password_hash=password_hash,
role=user_data.role,
is_active=True,
)
self.db.add(new_user)
await self.db.commit()
await self.db.refresh(new_user)
return new_user
async def update_user(self, user_id: int, user_data: UserUpdate) -> User:
"""Update user information"""
user = await self.get_by_id(user_id)
if not user:
raise NotFoundError("User not found")
if user_data.email and user_data.email != user.email:
existing = await self.get_by_email(user_data.email)
if existing and existing.id != user_id:
raise ConflictError("Email already registered")
user.email = user_data.email
if user_data.username and user_data.username != user.username:
existing = await self.get_by_username(user_data.username)
if existing and existing.id != user_id:
raise ConflictError("Username already taken")
user.username = user_data.username
if user_data.avatar_url is not None:
user.avatar_url = user_data.avatar_url
if user_data.role is not None:
user.role = user_data.role
if user_data.is_active is not None:
user.is_active = user_data.is_active
await self.db.commit()
await self.db.refresh(user)
return user
async def update_password(self, user_id: int, old_password: str, new_password: str) -> bool:
"""Update user password"""
from app.core.security import verify_password
user = await self.get_by_id(user_id)
if not user:
raise NotFoundError("User not found")
if not verify_password(old_password, user.password_hash):
raise PermissionDeniedError("Current password is incorrect")
if len(new_password) < 8:
raise ValueError("Password must be at least 8 characters long")
user.password_hash = hash_password(new_password)
await self.db.commit()
return True
async def delete_user(self, user_id: int) -> bool:
"""Delete a user"""
user = await self.get_by_id(user_id)
if not user:
raise NotFoundError("User not found")
await self.db.delete(user)
await self.db.commit()
return True
async def get_user_stats(self, user_id: int) -> UserStatsResponse:
"""Get user statistics"""
user = await self.get_by_id(user_id)
if not user:
raise NotFoundError("User not found")
return UserStatsResponse(
total_annotations=0,
completed_tasks=0,
reviewed_tasks=0,
weekly_annotation_count=0,
total_working_hours=0.0,
)
async def update_last_login(self, user_id: int) -> None:
"""Update user's last login timestamp"""
await self.db.execute(
update(User)
.where(User.id == user_id)
.values(last_login_at=func.now())
)
await self.db.commit()
"""
Celery tasks package
"""
from app.tasks.celery_app import celery_app
__all__ = ["celery_app"]
"""
AI inference tasks (run on GPU/CPU workers)
"""
import time
from app.tasks.celery_app import celery_app
@celery_app.task(bind=True, name="ai_tasks.run_auto_label")
def run_auto_label_task(self, auto_task_id: str):
from app.core.logging import get_logger
logger = get_logger()
logger.info(f"Starting auto-label task: {auto_task_id}")
try:
self.update_state(state="STARTED", meta={"progress": 0})
time.sleep(1)
self.update_state(state="PROGRESS", meta={"progress": 50})
time.sleep(1)
self.update_state(state="PROGRESS", meta={"progress": 100})
logger.info(f"Auto-label task completed: {auto_task_id}")
return {"status": "completed", "auto_task_id": auto_task_id}
except Exception as exc:
logger.error(f"Auto-label task failed: {auto_task_id} - {exc}")
return {"status": "failed", "auto_task_id": auto_task_id, "error": str(exc)}
@celery_app.task(name="ai_tasks.infer_yolo")
def infer_yolo(model_id: str, data_item_id: str):
"""YOLOv8 object detection inference"""
pass
@celery_app.task(name="ai_tasks.infer_sam")
def infer_sam(model_id: str, data_item_id: str):
"""SAM segmentation inference"""
pass
@celery_app.task(name="ai_tasks.infer_whisper")
def infer_whisper(model_id: str, data_item_id: str):
"""Whisper ASR inference"""
pass
@celery_app.task(name="ai_tasks.infer_ner")
def infer_ner(model_id: str, data_item_id: str):
"""NER inference"""
pass
"""
Celery application configuration
"""
from celery import Celery
from app.core.config import settings
celery_app = Celery(
"plabel",
broker=settings.celery_broker_url,
backend=settings.celery_result_backend,
include=[
"app.tasks.ai_tasks",
"app.tasks.export_tasks",
"app.tasks.file_tasks",
"app.tasks.cleanup_tasks",
],
)
celery_app.conf.update(
task_serializer="json",
accept_content=["json"],
result_serializer="json",
timezone="UTC",
enable_utc=True,
task_track_started=True,
task_acks_late=True,
task_reject_on_worker_lost=True,
worker_prefetch_multiplier=1,
task_routes={
"app.tasks.ai_tasks.*": {"queue": "gpu_tasks"},
"app.tasks.export_tasks.*": {"queue": "cpu_tasks"},
"app.tasks.file_tasks.*": {"queue": "cpu_tasks"},
"app.tasks.cleanup_tasks.*": {"queue": "cpu_tasks"},
},
)
__all__ = ["celery_app"]
"""
Scheduled cleanup tasks
"""
from app.tasks.celery_app import celery_app
@celery_app.task(name="cleanup_tasks.cleanup_expired_files")
def cleanup_expired_files():
from app.core.logging import get_logger
logger = get_logger()
logger.info("Running expired files cleanup")
return {"deleted": 0}
@celery_app.task(name="cleanup_tasks.cleanup_old_exports")
def cleanup_old_export_files():
from app.core.logging import get_logger
logger = get_logger()
logger.info("Running old exports cleanup")
return {"deleted": 0}
@celery_app.task(name="cleanup_tasks.cleanup_failed_tasks")
def cleanup_failed_tasks():
from app.core.logging import get_logger
logger = get_logger()
logger.info("Running failed tasks cleanup")
return {"deleted": 0}
"""
Data export tasks
"""
import time
from app.tasks.celery_app import celery_app
@celery_app.task(bind=True, name="export_tasks.run_export")
def run_export_task(self, export_id: str, task_id: str, format: str):
from app.core.logging import get_logger
logger = get_logger()
logger.info(f"Starting export task: {export_id} (format={format})")
try:
self.update_state(state="STARTED", meta={"progress": 0})
for i in range(1, 11):
time.sleep(0.3)
self.update_state(state="PROGRESS", meta={"progress": i * 10})
logger.info(f"Export task completed: {export_id}")
return {"status": "completed", "export_id": export_id}
except Exception as exc:
logger.error(f"Export task failed: {export_id} - {exc}")
return {"status": "failed", "export_id": export_id, "error": str(exc)}
"""
File processing tasks
"""
import time
from app.tasks.celery_app import celery_app
@celery_app.task(name="file_tasks.generate_thumbnail")
def generate_thumbnail(data_item_id: str):
from app.core.logging import get_logger
logger = get_logger()
logger.info(f"Generating thumbnail for: {data_item_id}")
time.sleep(0.5)
return {"data_item_id": data_item_id, "thumbnail_path": f"thumbnails/{data_item_id}.jpg"}
@celery_app.task(name="file_tasks.extract_metadata")
def extract_file_metadata(data_item_id: str):
from app.core.logging import get_logger
logger = get_logger()
logger.info(f"Extracting metadata for: {data_item_id}")
return {"data_item_id": data_item_id, "metadata": {}}
"""
Helper utilities
"""
import re
import uuid
from pathlib import Path
def generate_uuid() -> str:
return str(uuid.uuid4())
def get_file_extension(filename: str) -> str:
return Path(filename).suffix.lower()
def get_file_mime_type(filename: str) -> str:
ext = get_file_extension(filename)
mime_types = {
".jpg": "image/jpeg", ".jpeg": "image/jpeg",
".png": "image/png", ".webp": "image/webp",
".bmp": "image/bmp", ".gif": "image/gif",
".mp4": "video/mp4", ".webm": "video/webm",
".mov": "video/quicktime", ".avi": "video/x-msvideo",
".mp3": "audio/mpeg", ".wav": "audio/wav",
".ogg": "audio/ogg", ".m4a": "audio/mp4",
".txt": "text/plain", ".md": "text/markdown",
".json": "application/json",
}
return mime_types.get(ext, "application/octet-stream")
def sanitize_filename(filename: str) -> str:
safe = re.sub(r"[^\w\s.-]", "", filename)
safe = re.sub(r"\s+", "_", safe)
return safe[:255]
def format_file_size(size_bytes: int) -> str:
for unit in ["B", "KB", "MB", "GB", "TB"]:
if size_bytes < 1024:
return f"{size_bytes:.2f} {unit}"
size_bytes /= 1024
return f"{size_bytes:.2f} PB"
def paginate_query(page: int, page_size: int) -> tuple[int, int]:
offset = (page - 1) * page_size
return offset, page_size
"""
Standard API response utilities
"""
from typing import Any
def success_response(data: Any = None, message: str = "success", code: int = 200) -> dict:
return {"code": code, "message": message, "data": data}
def error_response(code: int, message: str, errors: list[dict] | None = None) -> dict:
resp = {"code": code, "message": message}
if errors:
resp["errors"] = errors
return resp
def paginated_response(
items: list,
total: int,
page: int,
page_size: int,
) -> dict:
total_pages = (total + page_size - 1) // page_size if page_size > 0 else 0
return {
"code": 200,
"message": "success",
"data": {
"items": items,
"total": total,
"page": page,
"page_size": page_size,
"total_pages": total_pages,
}
}
FROM python:3.11-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential libpq-dev libffi-dev libssl-dev ffmpeg \
libsm6 libxext6 libxrender-dev libgomp1 && \
rm -rf /var/lib/apt/lists/*
COPY requirements.txt /app/
RUN pip install --no-cache-dir -r requirements.txt
COPY . /app/
RUN mkdir -p /app/logs /app/data/uploads /app/data/thumbnails /app/data/exports
EXPOSE 8000
CMD ["gunicorn", "-w", "4", "-k", "uvicorn.workers.UvicornWorker", "app.main:app"]
-- Initial database setup
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
upstream backend {
server backend:8000;
}
server {
listen 80;
server_name localhost;
location /api/ {
proxy_pass http://backend;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /docs {
proxy_pass http://backend;
}
location /redoc {
proxy_pass http://backend;
}
}
}
version: '3.8'
services:
gpu-worker:
build:
context: .
dockerfile: Dockerfile.gpu
env_file:
- .env
volumes:
- ./app:/app/app
- ./models:/app/models
- ./data:/app/data
environment:
- CUDA_VISIBLE_DEVICES=0
- MODEL_DEVICE=cuda
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
command: celery -A app.tasks.celery_app worker -Q gpu_tasks -l info --concurrency=1
restart: unless-stopped
cpu-worker:
build:
context: .
dockerfile: Dockerfile
env_file:
- .env
volumes:
- ./app:/app/app
- ./models:/app/models
- ./data:/app/data
command: celery -A app.tasks.celery_app worker -Q cpu_tasks -l info --concurrency=4
restart: unless-stopped
version: '3.8'
services:
nginx:
image: nginx:alpine
ports:
- "80:80"
- "443:443"
volumes:
- ./configs/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./configs/nginx/ssl:/etc/nginx/ssl:ro
depends_on:
- backend
restart: unless-stopped
backend:
build:
context: .
dockerfile: configs/docker/Dockerfile.prod
env_file:
- .env
environment:
- DEBUG=false
- WORKERS=4
volumes:
- ./app:/app/app
- model_data:/app/models
deploy:
replicas: 2
restart: unless-stopped
celery_worker:
build:
context: .
dockerfile: configs/docker/Dockerfile.prod
env_file:
- .env
command: celery -A app.tasks.celery_app worker -Q gpu_tasks,cpu_tasks -l info --concurrency=4
volumes:
- ./app:/app/app
- model_data:/app/models
restart: unless-stopped
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
postgres:
image: postgres:15-alpine
env_file:
- .env
volumes:
- postgres_data:/var/lib/postgresql/data
- ./configs/docker/init.sql:/docker-entrypoint-initdb.d/init.sql
healthcheck:
test: ["CMD-SHELL", "pg_isready -U plabel"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped
redis:
image: redis:7-alpine
command: redis-server --appendonly yes
volumes:
- redis_data:/data
restart: unless-stopped
minio:
image: minio/minio:latest
env_file:
- .env
volumes:
- minio_data:/data
command: server /data --console-address ":9001"
restart: unless-stopped
backup:
image: postgres:15-alpine
env_file:
- .env
volumes:
- postgres_data:/data
- ./backups:/backups
command: >
sh -c 'while true; do
pg_dump -h postgres -U plabel plabel > /backups/backup_$(date +%Y%m%d_%H%M%S).sql;
sleep 86400;
done'
restart: unless-stopped
volumes:
postgres_data:
redis_data:
minio_data:
model_data:
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
差异被折叠。
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑!
请 注册 或者 后发表评论