Add organization and task queue features
- Introduced new models for `Organization` and `ReviewTask` to manage organizations and review tasks. - Implemented API endpoints for CRUD operations on organizations and tasks, including scanning organizations for repositories and PRs. - Developed a background worker for sequential processing of review tasks with priority handling and automatic retries. - Created frontend components for managing organizations and monitoring task queues, including real-time updates and filtering options. - Added comprehensive documentation for organization features and quick start guides. - Fixed UI issues and improved navigation for better user experience.
This commit is contained in:
Primakov Alexandr Alexandrovich
@@ -2,13 +2,15 @@
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api import repositories, reviews, webhooks
|
||||
from app.api import repositories, reviews, webhooks, organizations, tasks
|
||||
|
||||
api_router = APIRouter()
|
||||
|
||||
api_router.include_router(repositories.router, prefix="/repositories", tags=["repositories"])
|
||||
api_router.include_router(reviews.router, prefix="/reviews", tags=["reviews"])
|
||||
api_router.include_router(webhooks.router, prefix="/webhooks", tags=["webhooks"])
|
||||
api_router.include_router(organizations.router, prefix="/organizations", tags=["organizations"])
|
||||
api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
|
||||
|
||||
__all__ = ["api_router"]
|
||||
|
||||
|
||||
404
backend/app/api/organizations.py
Normal file
404
backend/app/api/organizations.py
Normal file
@@ -0,0 +1,404 @@
|
||||
"""Organizations API endpoints"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func
|
||||
from typing import List
|
||||
import secrets
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Organization, Repository, PullRequest, ReviewTask
|
||||
from app.schemas.organization import (
|
||||
OrganizationCreate,
|
||||
OrganizationUpdate,
|
||||
OrganizationResponse,
|
||||
OrganizationList,
|
||||
OrganizationScanResult
|
||||
)
|
||||
from app.utils import encrypt_token, decrypt_token
|
||||
from app.config import settings
|
||||
from app.services.gitea import GiteaService
|
||||
from app.services.github import GitHubService
|
||||
from app.services.bitbucket import BitbucketService
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("", response_model=OrganizationList)
|
||||
async def get_organizations(
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Get all organizations"""
|
||||
# Count total
|
||||
count_query = select(func.count(Organization.id))
|
||||
count_result = await db.execute(count_query)
|
||||
total = count_result.scalar()
|
||||
|
||||
# Get organizations
|
||||
query = select(Organization).order_by(Organization.created_at.desc()).offset(skip).limit(limit)
|
||||
result = await db.execute(query)
|
||||
organizations = result.scalars().all()
|
||||
|
||||
# Add webhook URLs
|
||||
items = []
|
||||
for org in organizations:
|
||||
webhook_url = f"http://{settings.host}:{settings.port}/api/webhooks/{org.platform.value}/org/{org.id}"
|
||||
items.append(OrganizationResponse(
|
||||
**org.__dict__,
|
||||
webhook_url=webhook_url
|
||||
))
|
||||
|
||||
return OrganizationList(items=items, total=total)
|
||||
|
||||
|
||||
@router.post("", response_model=OrganizationResponse)
|
||||
async def create_organization(
|
||||
organization: OrganizationCreate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Create a new organization"""
|
||||
# Generate webhook secret if not provided
|
||||
webhook_secret = organization.webhook_secret or secrets.token_urlsafe(32)
|
||||
|
||||
# Encrypt API token (если указан)
|
||||
encrypted_token = encrypt_token(organization.api_token) if organization.api_token else None
|
||||
|
||||
# Create organization
|
||||
db_organization = Organization(
|
||||
name=organization.name,
|
||||
platform=organization.platform,
|
||||
base_url=organization.base_url.rstrip('/'),
|
||||
api_token=encrypted_token,
|
||||
webhook_secret=webhook_secret,
|
||||
config=organization.config or {}
|
||||
)
|
||||
|
||||
db.add(db_organization)
|
||||
await db.commit()
|
||||
await db.refresh(db_organization)
|
||||
|
||||
# Prepare response
|
||||
webhook_url = f"http://{settings.host}:{settings.port}/api/webhooks/{db_organization.platform.value}/org/{db_organization.id}"
|
||||
|
||||
return OrganizationResponse(
|
||||
**db_organization.__dict__,
|
||||
webhook_url=webhook_url
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{organization_id}", response_model=OrganizationResponse)
|
||||
async def get_organization(
|
||||
organization_id: int,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Get organization by ID"""
|
||||
result = await db.execute(
|
||||
select(Organization).where(Organization.id == organization_id)
|
||||
)
|
||||
organization = result.scalar_one_or_none()
|
||||
|
||||
if not organization:
|
||||
raise HTTPException(status_code=404, detail="Organization not found")
|
||||
|
||||
webhook_url = f"http://{settings.host}:{settings.port}/api/webhooks/{organization.platform.value}/org/{organization.id}"
|
||||
|
||||
return OrganizationResponse(
|
||||
**organization.__dict__,
|
||||
webhook_url=webhook_url
|
||||
)
|
||||
|
||||
|
||||
@router.put("/{organization_id}", response_model=OrganizationResponse)
|
||||
async def update_organization(
|
||||
organization_id: int,
|
||||
organization_update: OrganizationUpdate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Update organization"""
|
||||
result = await db.execute(
|
||||
select(Organization).where(Organization.id == organization_id)
|
||||
)
|
||||
organization = result.scalar_one_or_none()
|
||||
|
||||
if not organization:
|
||||
raise HTTPException(status_code=404, detail="Organization not found")
|
||||
|
||||
# Update fields
|
||||
update_data = organization_update.model_dump(exclude_unset=True)
|
||||
|
||||
# Encrypt API token if provided and not empty
|
||||
if "api_token" in update_data and update_data["api_token"]:
|
||||
update_data["api_token"] = encrypt_token(update_data["api_token"])
|
||||
elif "api_token" in update_data and not update_data["api_token"]:
|
||||
# If empty string provided, don't update token
|
||||
del update_data["api_token"]
|
||||
|
||||
for field, value in update_data.items():
|
||||
setattr(organization, field, value)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(organization)
|
||||
|
||||
webhook_url = f"http://{settings.host}:{settings.port}/api/webhooks/{organization.platform.value}/org/{organization.id}"
|
||||
|
||||
return OrganizationResponse(
|
||||
**organization.__dict__,
|
||||
webhook_url=webhook_url
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{organization_id}")
|
||||
async def delete_organization(
|
||||
organization_id: int,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Delete organization"""
|
||||
result = await db.execute(
|
||||
select(Organization).where(Organization.id == organization_id)
|
||||
)
|
||||
organization = result.scalar_one_or_none()
|
||||
|
||||
if not organization:
|
||||
raise HTTPException(status_code=404, detail="Organization not found")
|
||||
|
||||
await db.delete(organization)
|
||||
await db.commit()
|
||||
|
||||
return {"message": "Organization deleted successfully"}
|
||||
|
||||
|
||||
@router.post("/{organization_id}/scan", response_model=OrganizationScanResult)
|
||||
async def scan_organization(
|
||||
organization_id: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Scan organization for repositories and PRs"""
|
||||
# Get organization
|
||||
result = await db.execute(
|
||||
select(Organization).where(Organization.id == organization_id)
|
||||
)
|
||||
organization = result.scalar_one_or_none()
|
||||
|
||||
if not organization:
|
||||
raise HTTPException(status_code=404, detail="Organization not found")
|
||||
|
||||
# Get API token
|
||||
if organization.api_token:
|
||||
try:
|
||||
api_token = decrypt_token(organization.api_token)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
else:
|
||||
# Use master token
|
||||
platform = organization.platform.value.lower()
|
||||
if platform == "gitea":
|
||||
api_token = settings.master_gitea_token
|
||||
elif platform == "github":
|
||||
api_token = settings.master_github_token
|
||||
elif platform == "bitbucket":
|
||||
api_token = settings.master_bitbucket_token
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported platform: {organization.platform}")
|
||||
|
||||
if not api_token:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"API token not set and master token for {platform} not configured"
|
||||
)
|
||||
|
||||
# Start scan
|
||||
scan_result = OrganizationScanResult(
|
||||
organization_id=organization_id,
|
||||
repositories_found=0,
|
||||
repositories_added=0,
|
||||
pull_requests_found=0,
|
||||
tasks_created=0,
|
||||
errors=[]
|
||||
)
|
||||
|
||||
try:
|
||||
if organization.platform.value == "gitea":
|
||||
await _scan_gitea_organization(organization, api_token, scan_result, db)
|
||||
elif organization.platform.value == "github":
|
||||
await _scan_github_organization(organization, api_token, scan_result, db)
|
||||
elif organization.platform.value == "bitbucket":
|
||||
await _scan_bitbucket_organization(organization, api_token, scan_result, db)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported platform: {organization.platform}")
|
||||
|
||||
# Update last scan time
|
||||
from datetime import datetime
|
||||
organization.last_scan_at = datetime.utcnow()
|
||||
await db.commit()
|
||||
|
||||
except Exception as e:
|
||||
scan_result.errors.append(str(e))
|
||||
raise HTTPException(status_code=500, detail=f"Scan failed: {str(e)}")
|
||||
|
||||
return scan_result
|
||||
|
||||
|
||||
async def _scan_gitea_organization(
|
||||
organization: Organization,
|
||||
api_token: str,
|
||||
scan_result: OrganizationScanResult,
|
||||
db: AsyncSession
|
||||
):
|
||||
"""Scan Gitea organization for repositories and PRs"""
|
||||
import httpx
|
||||
|
||||
headers = {
|
||||
"Authorization": f"token {api_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
# Get all repositories in organization
|
||||
url = f"{organization.base_url}/api/v1/orgs/{organization.name}/repos"
|
||||
|
||||
print(f"\n🔍 Сканирование организации {organization.name} на {organization.base_url}")
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
repos = response.json()
|
||||
|
||||
scan_result.repositories_found = len(repos)
|
||||
print(f" Найдено репозиториев: {len(repos)}")
|
||||
|
||||
for repo_data in repos:
|
||||
repo_name = repo_data["name"]
|
||||
repo_owner = repo_data["owner"]["login"]
|
||||
repo_url = repo_data["html_url"]
|
||||
|
||||
# Check if repository already exists
|
||||
existing_repo = await db.execute(
|
||||
select(Repository).where(Repository.url == repo_url)
|
||||
)
|
||||
repository = existing_repo.scalar_one_or_none()
|
||||
|
||||
if not repository:
|
||||
# Create new repository
|
||||
repository = Repository(
|
||||
name=f"{repo_owner}/{repo_name}",
|
||||
platform=organization.platform,
|
||||
url=repo_url,
|
||||
api_token=organization.api_token, # Use same token as org
|
||||
webhook_secret=organization.webhook_secret,
|
||||
config=organization.config
|
||||
)
|
||||
db.add(repository)
|
||||
await db.flush()
|
||||
scan_result.repositories_added += 1
|
||||
print(f" ✅ Добавлен репозиторий: {repo_owner}/{repo_name}")
|
||||
|
||||
# Scan PRs in this repository
|
||||
await _scan_repository_prs(
|
||||
repository,
|
||||
organization.base_url,
|
||||
repo_owner,
|
||||
repo_name,
|
||||
api_token,
|
||||
scan_result,
|
||||
db
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def _scan_github_organization(
|
||||
organization: Organization,
|
||||
api_token: str,
|
||||
scan_result: OrganizationScanResult,
|
||||
db: AsyncSession
|
||||
):
|
||||
"""Scan GitHub organization"""
|
||||
# TODO: Implement GitHub org scanning
|
||||
scan_result.errors.append("GitHub organization scanning not yet implemented")
|
||||
|
||||
|
||||
async def _scan_bitbucket_organization(
|
||||
organization: Organization,
|
||||
api_token: str,
|
||||
scan_result: OrganizationScanResult,
|
||||
db: AsyncSession
|
||||
):
|
||||
"""Scan Bitbucket organization"""
|
||||
# TODO: Implement Bitbucket org scanning
|
||||
scan_result.errors.append("Bitbucket organization scanning not yet implemented")
|
||||
|
||||
|
||||
async def _scan_repository_prs(
|
||||
repository: Repository,
|
||||
base_url: str,
|
||||
owner: str,
|
||||
repo: str,
|
||||
api_token: str,
|
||||
scan_result: OrganizationScanResult,
|
||||
db: AsyncSession
|
||||
):
|
||||
"""Scan repository for open PRs and create tasks"""
|
||||
import httpx
|
||||
|
||||
headers = {
|
||||
"Authorization": f"token {api_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
# Get open PRs
|
||||
url = f"{base_url}/api/v1/repos/{owner}/{repo}/pulls?state=open"
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
prs = response.json()
|
||||
|
||||
for pr_data in prs:
|
||||
pr_number = pr_data["number"]
|
||||
scan_result.pull_requests_found += 1
|
||||
|
||||
# Check if PR already exists
|
||||
existing_pr = await db.execute(
|
||||
select(PullRequest).where(
|
||||
PullRequest.repository_id == repository.id,
|
||||
PullRequest.pr_number == pr_number
|
||||
)
|
||||
)
|
||||
pull_request = existing_pr.scalar_one_or_none()
|
||||
|
||||
if not pull_request:
|
||||
# Create new PR
|
||||
pull_request = PullRequest(
|
||||
repository_id=repository.id,
|
||||
pr_number=pr_number,
|
||||
title=pr_data["title"],
|
||||
author=pr_data["user"]["login"],
|
||||
source_branch=pr_data["head"]["ref"],
|
||||
target_branch=pr_data["base"]["ref"],
|
||||
url=pr_data["html_url"],
|
||||
status="OPEN"
|
||||
)
|
||||
db.add(pull_request)
|
||||
await db.flush()
|
||||
|
||||
# Check if task already exists for this PR
|
||||
existing_task = await db.execute(
|
||||
select(ReviewTask).where(
|
||||
ReviewTask.pull_request_id == pull_request.id,
|
||||
ReviewTask.status.in_(["pending", "in_progress"])
|
||||
)
|
||||
)
|
||||
task = existing_task.scalar_one_or_none()
|
||||
|
||||
if not task:
|
||||
# Create review task
|
||||
task = ReviewTask(
|
||||
pull_request_id=pull_request.id,
|
||||
priority="normal"
|
||||
)
|
||||
db.add(task)
|
||||
scan_result.tasks_created += 1
|
||||
print(f" 📝 Создана задача для PR #{pr_number}: {pr_data['title']}")
|
||||
|
||||
197
backend/app/api/tasks.py
Normal file
197
backend/app/api/tasks.py
Normal file
@@ -0,0 +1,197 @@
|
||||
"""Task Queue API endpoints"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func
|
||||
from typing import List
|
||||
from pydantic import BaseModel
|
||||
from datetime import datetime
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import ReviewTask, PullRequest
|
||||
from app.models.review_task import TaskStatusEnum, TaskPriorityEnum
|
||||
from app.workers.task_worker import get_worker
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class TaskResponse(BaseModel):
|
||||
"""Task response schema"""
|
||||
id: int
|
||||
pull_request_id: int
|
||||
pr_number: int | None
|
||||
pr_title: str | None
|
||||
status: TaskStatusEnum
|
||||
priority: TaskPriorityEnum
|
||||
created_at: datetime
|
||||
started_at: datetime | None
|
||||
completed_at: datetime | None
|
||||
error_message: str | None
|
||||
retry_count: int
|
||||
max_retries: int
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class TaskListResponse(BaseModel):
|
||||
"""Task list response"""
|
||||
items: List[TaskResponse]
|
||||
total: int
|
||||
pending: int
|
||||
in_progress: int
|
||||
completed: int
|
||||
failed: int
|
||||
|
||||
|
||||
class WorkerStatusResponse(BaseModel):
|
||||
"""Worker status response"""
|
||||
running: bool
|
||||
current_task_id: int | None
|
||||
poll_interval: int
|
||||
|
||||
|
||||
@router.get("", response_model=TaskListResponse)
|
||||
async def get_tasks(
|
||||
status: TaskStatusEnum | None = None,
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Get all tasks"""
|
||||
# Count total
|
||||
count_query = select(func.count(ReviewTask.id))
|
||||
if status:
|
||||
count_query = count_query.where(ReviewTask.status == status)
|
||||
count_result = await db.execute(count_query)
|
||||
total = count_result.scalar()
|
||||
|
||||
# Count by status
|
||||
pending_count = await db.execute(
|
||||
select(func.count(ReviewTask.id)).where(ReviewTask.status == TaskStatusEnum.PENDING)
|
||||
)
|
||||
in_progress_count = await db.execute(
|
||||
select(func.count(ReviewTask.id)).where(ReviewTask.status == TaskStatusEnum.IN_PROGRESS)
|
||||
)
|
||||
completed_count = await db.execute(
|
||||
select(func.count(ReviewTask.id)).where(ReviewTask.status == TaskStatusEnum.COMPLETED)
|
||||
)
|
||||
failed_count = await db.execute(
|
||||
select(func.count(ReviewTask.id)).where(ReviewTask.status == TaskStatusEnum.FAILED)
|
||||
)
|
||||
|
||||
# Get tasks with PR info
|
||||
query = select(ReviewTask, PullRequest).join(
|
||||
PullRequest, ReviewTask.pull_request_id == PullRequest.id
|
||||
).order_by(ReviewTask.created_at.desc())
|
||||
|
||||
if status:
|
||||
query = query.where(ReviewTask.status == status)
|
||||
|
||||
query = query.offset(skip).limit(limit)
|
||||
result = await db.execute(query)
|
||||
rows = result.all()
|
||||
|
||||
items = []
|
||||
for task, pr in rows:
|
||||
items.append(TaskResponse(
|
||||
id=task.id,
|
||||
pull_request_id=task.pull_request_id,
|
||||
pr_number=pr.pr_number,
|
||||
pr_title=pr.title,
|
||||
status=task.status,
|
||||
priority=task.priority,
|
||||
created_at=task.created_at,
|
||||
started_at=task.started_at,
|
||||
completed_at=task.completed_at,
|
||||
error_message=task.error_message,
|
||||
retry_count=task.retry_count,
|
||||
max_retries=task.max_retries
|
||||
))
|
||||
|
||||
return TaskListResponse(
|
||||
items=items,
|
||||
total=total,
|
||||
pending=pending_count.scalar(),
|
||||
in_progress=in_progress_count.scalar(),
|
||||
completed=completed_count.scalar(),
|
||||
failed=failed_count.scalar()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/worker/status", response_model=WorkerStatusResponse)
|
||||
async def get_worker_status():
|
||||
"""Get worker status"""
|
||||
worker = get_worker()
|
||||
|
||||
if not worker:
|
||||
return WorkerStatusResponse(
|
||||
running=False,
|
||||
current_task_id=None,
|
||||
poll_interval=0
|
||||
)
|
||||
|
||||
return WorkerStatusResponse(
|
||||
running=worker.running,
|
||||
current_task_id=worker.current_task_id,
|
||||
poll_interval=worker.poll_interval
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/retry")
|
||||
async def retry_task(
|
||||
task_id: int,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Retry failed task"""
|
||||
result = await db.execute(
|
||||
select(ReviewTask).where(ReviewTask.id == task_id)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if not task:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
|
||||
if task.status not in [TaskStatusEnum.FAILED, TaskStatusEnum.COMPLETED]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Cannot retry task with status: {task.status}"
|
||||
)
|
||||
|
||||
# Reset task
|
||||
task.status = TaskStatusEnum.PENDING
|
||||
task.error_message = None
|
||||
task.retry_count = 0
|
||||
task.started_at = None
|
||||
task.completed_at = None
|
||||
|
||||
await db.commit()
|
||||
|
||||
return {"message": "Task queued for retry"}
|
||||
|
||||
|
||||
@router.delete("/{task_id}")
|
||||
async def delete_task(
|
||||
task_id: int,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Delete task"""
|
||||
result = await db.execute(
|
||||
select(ReviewTask).where(ReviewTask.id == task_id)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if not task:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
|
||||
if task.status == TaskStatusEnum.IN_PROGRESS:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot delete task that is in progress"
|
||||
)
|
||||
|
||||
await db.delete(task)
|
||||
await db.commit()
|
||||
|
||||
return {"message": "Task deleted"}
|
||||
|
||||
@@ -45,9 +45,16 @@ async def lifespan(app: FastAPI):
|
||||
"""Lifespan events"""
|
||||
# Startup
|
||||
await init_db()
|
||||
|
||||
# Start task worker
|
||||
from app.workers.task_worker import start_worker
|
||||
await start_worker()
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
pass
|
||||
from app.workers.task_worker import stop_worker
|
||||
await stop_worker()
|
||||
|
||||
|
||||
# Create FastAPI app
|
||||
|
||||
@@ -4,6 +4,8 @@ from app.models.repository import Repository
|
||||
from app.models.pull_request import PullRequest
|
||||
from app.models.review import Review
|
||||
from app.models.comment import Comment
|
||||
from app.models.organization import Organization
|
||||
from app.models.review_task import ReviewTask
|
||||
|
||||
__all__ = ["Repository", "PullRequest", "Review", "Comment"]
|
||||
__all__ = ["Repository", "PullRequest", "Review", "Comment", "Organization", "ReviewTask"]
|
||||
|
||||
|
||||
38
backend/app/models/organization.py
Normal file
38
backend/app/models/organization.py
Normal file
@@ -0,0 +1,38 @@
|
||||
"""Organization model"""
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Boolean, DateTime, JSON, Enum
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
from datetime import datetime
|
||||
import enum
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class OrganizationPlatformEnum(str, enum.Enum):
|
||||
"""Git platform types"""
|
||||
GITEA = "gitea"
|
||||
GITHUB = "github"
|
||||
BITBUCKET = "bitbucket"
|
||||
|
||||
|
||||
class Organization(Base):
|
||||
"""Organization model for tracking Git organizations"""
|
||||
|
||||
__tablename__ = "organizations"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String, nullable=False) # Имя организации
|
||||
platform = Column(Enum(OrganizationPlatformEnum), nullable=False)
|
||||
base_url = Column(String, nullable=False) # https://git.example.com
|
||||
api_token = Column(String, nullable=True) # Encrypted, optional (uses master token)
|
||||
webhook_secret = Column(String, nullable=False)
|
||||
config = Column(JSON, default=dict) # Review configuration
|
||||
is_active = Column(Boolean, default=True)
|
||||
last_scan_at = Column(DateTime, nullable=True) # Когда последний раз сканировали
|
||||
created_at = Column(DateTime, default=datetime.utcnow, server_default=func.now())
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, server_default=func.now())
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Organization(id={self.id}, name={self.name}, platform={self.platform})>"
|
||||
|
||||
52
backend/app/models/review_task.py
Normal file
52
backend/app/models/review_task.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""Review Task Queue model"""
|
||||
|
||||
from sqlalchemy import Column, Integer, String, DateTime, Enum, ForeignKey
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
from datetime import datetime
|
||||
import enum
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class TaskStatusEnum(str, enum.Enum):
|
||||
"""Task status types"""
|
||||
PENDING = "pending" # В очереди
|
||||
IN_PROGRESS = "in_progress" # Выполняется
|
||||
COMPLETED = "completed" # Завершено
|
||||
FAILED = "failed" # Ошибка
|
||||
|
||||
|
||||
class TaskPriorityEnum(str, enum.Enum):
|
||||
"""Task priority types"""
|
||||
LOW = "low"
|
||||
NORMAL = "normal"
|
||||
HIGH = "high"
|
||||
|
||||
|
||||
class ReviewTask(Base):
|
||||
"""Review task queue for sequential processing"""
|
||||
|
||||
__tablename__ = "review_tasks"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
pull_request_id = Column(Integer, ForeignKey("pull_requests.id"), nullable=False)
|
||||
status = Column(Enum(TaskStatusEnum), default=TaskStatusEnum.PENDING, nullable=False, index=True)
|
||||
priority = Column(Enum(TaskPriorityEnum), default=TaskPriorityEnum.NORMAL, nullable=False)
|
||||
|
||||
# Tracking
|
||||
created_at = Column(DateTime, default=datetime.utcnow, server_default=func.now())
|
||||
started_at = Column(DateTime, nullable=True) # Когда началась обработка
|
||||
completed_at = Column(DateTime, nullable=True) # Когда завершилась
|
||||
error_message = Column(String, nullable=True)
|
||||
|
||||
# Retry logic
|
||||
retry_count = Column(Integer, default=0)
|
||||
max_retries = Column(Integer, default=3)
|
||||
|
||||
# Relationships
|
||||
pull_request = relationship("PullRequest", backref="review_tasks")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<ReviewTask(id={self.id}, pr_id={self.pull_request_id}, status={self.status})>"
|
||||
|
||||
60
backend/app/schemas/organization.py
Normal file
60
backend/app/schemas/organization.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""Organization schemas"""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional, Dict, Any, List
|
||||
from datetime import datetime
|
||||
from app.models.organization import OrganizationPlatformEnum
|
||||
|
||||
|
||||
class OrganizationBase(BaseModel):
|
||||
"""Base organization schema"""
|
||||
name: str = Field(..., description="Organization name")
|
||||
platform: OrganizationPlatformEnum = Field(..., description="Git platform")
|
||||
base_url: str = Field(..., description="Base URL (e.g., https://git.example.com)")
|
||||
config: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Review configuration")
|
||||
|
||||
|
||||
class OrganizationCreate(OrganizationBase):
|
||||
"""Schema for creating organization"""
|
||||
api_token: Optional[str] = Field(None, description="API token (optional, uses master token if not set)")
|
||||
webhook_secret: Optional[str] = Field(None, description="Webhook secret (generated if not provided)")
|
||||
|
||||
|
||||
class OrganizationUpdate(BaseModel):
|
||||
"""Schema for updating organization"""
|
||||
name: Optional[str] = None
|
||||
base_url: Optional[str] = None
|
||||
api_token: Optional[str] = None
|
||||
webhook_secret: Optional[str] = None
|
||||
config: Optional[Dict[str, Any]] = None
|
||||
is_active: Optional[bool] = None
|
||||
|
||||
|
||||
class OrganizationResponse(OrganizationBase):
|
||||
"""Schema for organization response"""
|
||||
id: int
|
||||
is_active: bool
|
||||
last_scan_at: Optional[datetime]
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
webhook_url: str = Field(..., description="Webhook URL for this organization")
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class OrganizationList(BaseModel):
|
||||
"""Schema for organization list response"""
|
||||
items: List[OrganizationResponse]
|
||||
total: int
|
||||
|
||||
|
||||
class OrganizationScanResult(BaseModel):
|
||||
"""Schema for organization scan result"""
|
||||
organization_id: int
|
||||
repositories_found: int
|
||||
repositories_added: int
|
||||
pull_requests_found: int
|
||||
tasks_created: int
|
||||
errors: List[str] = []
|
||||
|
||||
199
backend/app/workers/task_worker.py
Normal file
199
backend/app/workers/task_worker.py
Normal file
@@ -0,0 +1,199 @@
|
||||
"""Task Worker for sequential review processing"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import ReviewTask, PullRequest, Repository, Review
|
||||
from app.models.review_task import TaskStatusEnum
|
||||
from app.agents.reviewer import CodeReviewAgent
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReviewTaskWorker:
|
||||
"""Worker that processes review tasks sequentially"""
|
||||
|
||||
def __init__(self):
|
||||
self.running = False
|
||||
self.current_task_id = None
|
||||
self.poll_interval = 10 # секунд между проверками
|
||||
|
||||
async def start(self):
|
||||
"""Start the worker"""
|
||||
self.running = True
|
||||
logger.info("🚀 Task Worker запущен")
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
await self._process_next_task()
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Ошибка в Task Worker: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Подождать перед следующей проверкой
|
||||
await asyncio.sleep(self.poll_interval)
|
||||
|
||||
async def stop(self):
|
||||
"""Stop the worker"""
|
||||
self.running = False
|
||||
logger.info("⏹️ Task Worker остановлен")
|
||||
|
||||
async def _process_next_task(self):
|
||||
"""Process next pending task"""
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Проверяем есть ли уже выполняющаяся задача
|
||||
in_progress_query = select(ReviewTask).where(
|
||||
ReviewTask.status == TaskStatusEnum.IN_PROGRESS
|
||||
)
|
||||
result = await db.execute(in_progress_query)
|
||||
in_progress = result.scalar_one_or_none()
|
||||
|
||||
if in_progress:
|
||||
# Уже есть задача в работе, ждем
|
||||
logger.debug(f"⏳ Задача #{in_progress.id} уже выполняется")
|
||||
return
|
||||
|
||||
# Берем следующую pending задачу (с приоритетом)
|
||||
pending_query = select(ReviewTask).where(
|
||||
ReviewTask.status == TaskStatusEnum.PENDING
|
||||
).order_by(
|
||||
ReviewTask.priority.desc(), # HIGH > NORMAL > LOW
|
||||
ReviewTask.created_at.asc() # Старые первыми
|
||||
).limit(1)
|
||||
|
||||
result = await db.execute(pending_query)
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if not task:
|
||||
# Нет задач в очереди
|
||||
return
|
||||
|
||||
logger.info(f"\n{'='*80}")
|
||||
logger.info(f"📋 Начало обработки задачи #{task.id}")
|
||||
logger.info(f" PR ID: {task.pull_request_id}")
|
||||
logger.info(f" Приоритет: {task.priority}")
|
||||
logger.info(f"={'='*80}\n")
|
||||
|
||||
# Отмечаем задачу как in_progress
|
||||
task.status = TaskStatusEnum.IN_PROGRESS
|
||||
task.started_at = datetime.utcnow()
|
||||
self.current_task_id = task.id
|
||||
await db.commit()
|
||||
|
||||
try:
|
||||
# Выполняем review
|
||||
await self._execute_review(task, db)
|
||||
|
||||
# Успешно завершено
|
||||
task.status = TaskStatusEnum.COMPLETED
|
||||
task.completed_at = datetime.utcnow()
|
||||
logger.info(f"✅ Задача #{task.id} успешно завершена")
|
||||
|
||||
except Exception as e:
|
||||
# Ошибка при выполнении
|
||||
task.retry_count += 1
|
||||
task.error_message = str(e)
|
||||
|
||||
if task.retry_count >= task.max_retries:
|
||||
# Превышено количество попыток
|
||||
task.status = TaskStatusEnum.FAILED
|
||||
task.completed_at = datetime.utcnow()
|
||||
logger.error(f"❌ Задача #{task.id} провалена после {task.retry_count} попыток: {e}")
|
||||
else:
|
||||
# Вернуть в pending для повторной попытки
|
||||
task.status = TaskStatusEnum.PENDING
|
||||
logger.warning(f"⚠️ Задача #{task.id} вернулась в очередь (попытка {task.retry_count}/{task.max_retries}): {e}")
|
||||
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
finally:
|
||||
self.current_task_id = None
|
||||
await db.commit()
|
||||
|
||||
async def _execute_review(self, task: ReviewTask, db: AsyncSession):
|
||||
"""Execute review for the task"""
|
||||
# Get PR with repository
|
||||
result = await db.execute(
|
||||
select(PullRequest).where(PullRequest.id == task.pull_request_id)
|
||||
)
|
||||
pull_request = result.scalar_one_or_none()
|
||||
|
||||
if not pull_request:
|
||||
raise ValueError(f"PullRequest {task.pull_request_id} not found")
|
||||
|
||||
# Get repository
|
||||
result = await db.execute(
|
||||
select(Repository).where(Repository.id == pull_request.repository_id)
|
||||
)
|
||||
repository = result.scalar_one_or_none()
|
||||
|
||||
if not repository:
|
||||
raise ValueError(f"Repository {pull_request.repository_id} not found")
|
||||
|
||||
# Check if review already exists and is not failed
|
||||
existing_review = await db.execute(
|
||||
select(Review).where(
|
||||
Review.pull_request_id == pull_request.id
|
||||
).order_by(Review.started_at.desc())
|
||||
)
|
||||
review = existing_review.scalar_one_or_none()
|
||||
|
||||
if review and review.status not in ["failed", "pending"]:
|
||||
logger.info(f" Review already exists with status: {review.status}")
|
||||
return
|
||||
|
||||
# Create new review if doesn't exist
|
||||
if not review:
|
||||
review = Review(
|
||||
pull_request_id=pull_request.id,
|
||||
status="pending"
|
||||
)
|
||||
db.add(review)
|
||||
await db.commit()
|
||||
await db.refresh(review)
|
||||
|
||||
# Run review agent
|
||||
logger.info(f" 🤖 Запуск AI review для PR #{pull_request.pr_number}")
|
||||
|
||||
agent = CodeReviewAgent(db)
|
||||
await agent.review_pull_request(
|
||||
repository_id=repository.id,
|
||||
pr_number=pull_request.pr_number,
|
||||
review_id=review.id
|
||||
)
|
||||
|
||||
logger.info(f" ✅ Review завершен для PR #{pull_request.pr_number}")
|
||||
|
||||
|
||||
# Global worker instance
|
||||
_worker_instance: ReviewTaskWorker | None = None
|
||||
|
||||
|
||||
async def start_worker():
|
||||
"""Start the global worker instance"""
|
||||
global _worker_instance
|
||||
if _worker_instance is None:
|
||||
_worker_instance = ReviewTaskWorker()
|
||||
# Запускаем в фоне
|
||||
asyncio.create_task(_worker_instance.start())
|
||||
|
||||
|
||||
async def stop_worker():
|
||||
"""Stop the global worker instance"""
|
||||
global _worker_instance
|
||||
if _worker_instance:
|
||||
await _worker_instance.stop()
|
||||
_worker_instance = None
|
||||
|
||||
|
||||
def get_worker() -> ReviewTaskWorker | None:
|
||||
"""Get the current worker instance"""
|
||||
return _worker_instance
|
||||
|
||||
27
backend/migrate.py
Normal file
27
backend/migrate.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Простой скрипт для создания таблиц в БД
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from app.database import engine, Base
|
||||
from app.models import Organization, ReviewTask, Repository, PullRequest, Review, Comment
|
||||
|
||||
|
||||
async def create_tables():
|
||||
"""Создать все таблицы"""
|
||||
async with engine.begin() as conn:
|
||||
# Создать все таблицы
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
print("✅ Таблицы созданы успешно!")
|
||||
print(" - organizations")
|
||||
print(" - review_tasks")
|
||||
print(" - repositories")
|
||||
print(" - pull_requests")
|
||||
print(" - reviews")
|
||||
print(" - comments")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(create_tables())
|
||||
|
||||
Reference in New Issue
Block a user