baseline: Clover 独立仓库首次基线提交
将 Clover 从上层产品包旧仓库中独立出来,建立专属版本控制。 当前状态=纵切片端到端已打通(登录→选品→出文出图→审核→下载包), M1文案质量去套路化已验收。此提交作为后续按核销清单逐条修复的基线。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
11
backend/app/workers/README.md
Normal file
11
backend/app/workers/README.md
Normal file
@@ -0,0 +1,11 @@
|
||||
# app/workers/
|
||||
|
||||
Celery worker 占位:
|
||||
- celery_app.py # Celery实例配置(broker=Redis)
|
||||
- task_runner.py # 主任务:只接收task_id → 查库→FERNET_KEY解密key → 调模型
|
||||
# 铁律:明文key绝不进Celery参数,只在函数局部变量,不落盘不打日志
|
||||
- subtasks/
|
||||
analyze.py # 分析标杆笔记8特征
|
||||
generate_text.py # 文案双轨(轨A一次5角度JSON/轨B跳过)
|
||||
generate_image.py # 并发生图asyncio.gather(A/B/C三策略)
|
||||
postprocess.py # 去水印后处理
|
||||
0
backend/app/workers/__init__.py
Normal file
0
backend/app/workers/__init__.py
Normal file
36
backend/app/workers/celery_app.py
Normal file
36
backend/app/workers/celery_app.py
Normal file
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
app/workers/celery_app.py — Celery 任务框架壳
|
||||
铁律:只传 task_id,绝不传 key(基石B)。
|
||||
worker 内查库 → Fernet 解密 → 局部变量,不落盘不打日志。
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from celery import Celery
|
||||
|
||||
from app.core.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
celery_app = Celery(
|
||||
"clover",
|
||||
broker=settings.celery_broker(),
|
||||
backend=settings.celery_backend(),
|
||||
include=["app.workers.tasks", "app.workers.replenish_task"],
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
result_serializer="json",
|
||||
accept_content=["json"],
|
||||
timezone="Asia/Shanghai",
|
||||
enable_utc=True,
|
||||
task_track_started=True,
|
||||
task_acks_late=True, # 任务处理完才 ACK,防丢失
|
||||
worker_prefetch_multiplier=1, # 一次只取1条,防长任务堆积
|
||||
task_routes={
|
||||
"app.workers.tasks.run_generation_pipeline": {"queue": "generation"},
|
||||
"app.workers.tasks.build_delivery_package": {"queue": "packaging"},
|
||||
},
|
||||
)
|
||||
104
backend/app/workers/packaging_task.py
Normal file
104
backend/app/workers/packaging_task.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""
|
||||
app/workers/packaging_task.py — 交付打包 Celery 任务
|
||||
build_delivery_package:查已选文案+图片 → package_exporter → 存路径
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from app.workers.celery_app import celery_app
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_db():
|
||||
from app.core.database import SessionLocal
|
||||
return SessionLocal()
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.workers.tasks.build_delivery_package",
|
||||
max_retries=2,
|
||||
default_retry_delay=10,
|
||||
queue="packaging",
|
||||
)
|
||||
def build_delivery_package(self, package_id: int) -> dict:
|
||||
"""打包交付任务。查 delivery_packages → 收集笔记 → package_exporter"""
|
||||
logger.info("build_delivery_package start: package_id=%s", package_id)
|
||||
db = _get_db()
|
||||
try:
|
||||
from app.models.task import DeliveryPackage, TextCandidate, ImageCandidate
|
||||
from app.constants.enums import PackageStatus
|
||||
|
||||
pkg = db.query(DeliveryPackage).filter(DeliveryPackage.id == package_id).first()
|
||||
if not pkg:
|
||||
raise ValueError(f"package_id={package_id} not found")
|
||||
|
||||
workspace_id = pkg.workspace_id
|
||||
task_id = pkg.task_id
|
||||
|
||||
from app.core.config import get_settings
|
||||
settings = get_settings()
|
||||
upload_base = settings.UPLOAD_BASE_PATH.rstrip("/")
|
||||
|
||||
selected_text = db.query(TextCandidate).filter(
|
||||
TextCandidate.task_id == task_id, TextCandidate.is_selected == True,
|
||||
).first()
|
||||
# 整套全打(倩倩姐2026-06-08拍板):一条笔记的全部图按 seq 排序进包,
|
||||
# 不再只打 is_selected 的封面。北哥6张标准套 seq=1 是 hook 封面,天然排第一。
|
||||
selected_images = db.query(ImageCandidate).filter(
|
||||
ImageCandidate.task_id == task_id,
|
||||
).order_by(ImageCandidate.seq).all()
|
||||
|
||||
if not selected_text:
|
||||
raise ValueError("无已选文案,请先选择文案")
|
||||
|
||||
text_data = json.loads(selected_text.content or "{}")
|
||||
images_data = []
|
||||
for ic in selected_images:
|
||||
img_bytes = b""
|
||||
if ic.url:
|
||||
# url 形如 /uploads/ws/task/file.jpg,本身已含 uploads 前缀。
|
||||
# 工作目录是 /app,直接 lstrip("/") 当相对路径读,不能再拼 upload_base(会重复 uploads/uploads)。
|
||||
rel = ic.url.lstrip("/")
|
||||
abs_path = rel
|
||||
try:
|
||||
with open(abs_path, "rb") as f:
|
||||
img_bytes = f.read()
|
||||
except OSError as e:
|
||||
logger.warning("图片读取失败,跳过:%s %s", abs_path, e)
|
||||
images_data.append({
|
||||
"seq": ic.seq,
|
||||
"role": ic.role.value if hasattr(ic.role, "value") else str(ic.role),
|
||||
"data": img_bytes,
|
||||
})
|
||||
|
||||
notes = [{
|
||||
"title": text_data.get("title", ""),
|
||||
"content": text_data.get("content", ""),
|
||||
"tags": text_data.get("tags", []),
|
||||
"images": images_data,
|
||||
"banned_word_status": (selected_text.banned_word_status.value
|
||||
if hasattr(selected_text.banned_word_status, "value")
|
||||
else str(selected_text.banned_word_status)),
|
||||
}]
|
||||
|
||||
from app.services.ai_engine.package_exporter import build_delivery_package as do_build
|
||||
# 打包产物放专用目录 uploads/packages/,与图片目录 uploads/{ws}/{task}/ 分开
|
||||
packages_base = f"{upload_base}/packages"
|
||||
zip_path = do_build(workspace_id, task_id, notes, base_path=packages_base)
|
||||
|
||||
pkg.package_path = zip_path
|
||||
pkg.download_url = f"/api/v1/delivery-packages/{package_id}/download-file"
|
||||
pkg.status = PackageStatus.READY
|
||||
db.commit()
|
||||
|
||||
logger.info("delivery package ready: package_id=%s path=%s", package_id, zip_path)
|
||||
return {"package_id": package_id, "status": "ready", "path": zip_path}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("build_delivery_package failed: package_id=%s err=%s", package_id, exc)
|
||||
raise self.retry(exc=exc)
|
||||
finally:
|
||||
db.close()
|
||||
298
backend/app/workers/pipeline_io.py
Normal file
298
backend/app/workers/pipeline_io.py
Normal file
@@ -0,0 +1,298 @@
|
||||
"""
|
||||
app/workers/pipeline_io.py — 生产链 Step5-8
|
||||
Step5: 文案生成(generate_text_variants)
|
||||
Step6: 图片生成(generate_storyboard_images,asyncio.gather)
|
||||
Step7: 图片后处理(image_postprocessor)
|
||||
Step8: 存 text_candidates / image_candidates → 更新状态 → 推 task_done
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_image_path(img_path: str) -> str:
|
||||
"""
|
||||
解析产品参考图路径,兼容绝对路径(新)与历史相对路径(旧)。
|
||||
新数据存绝对路径(/app/uploads/...)直接返回;
|
||||
旧数据存相对路径(uploads/packages/...)锚定到 UPLOAD_ABS_ROOT 的父级,
|
||||
避免 worker(cwd=/) 解析失败。
|
||||
"""
|
||||
if not img_path:
|
||||
return ""
|
||||
if os.path.isabs(img_path):
|
||||
return img_path
|
||||
from app.core.config import get_settings
|
||||
# UPLOAD_ABS_ROOT=/app/uploads,其父级 /app 是相对路径(uploads/...)的锚点
|
||||
root_parent = os.path.dirname(get_settings().UPLOAD_ABS_ROOT.rstrip("/"))
|
||||
return os.path.join(root_parent, img_path)
|
||||
|
||||
|
||||
def run_text_generation(db, clients, task, product_dict: dict, flywheel_fragment: str,
|
||||
push_fn, workspace_id: int, seq_start: int) -> tuple[list, int, bool]:
|
||||
"""
|
||||
Step5: 调 generate_text_variants → 存 TextCandidate → 推 SSE → 写 ai_call_logs。
|
||||
S1: 存库前过滤——只存 passed且score>=90且banned_word_status!='hard_block' 的文案。
|
||||
合格数 < task.text_count 时 needs_replenish=True(由主任务发起后台补充子任务)。
|
||||
返回 (candidates_raw, next_seq, needs_replenish)。
|
||||
"""
|
||||
import time
|
||||
from app.services.ai_engine.text_variants import generate_text_variants
|
||||
from app.models.product import BannedWord
|
||||
from app.models.task import TextCandidate
|
||||
from app.models.flywheel import AiCallLog
|
||||
from app.models.workspace import UserApiKey
|
||||
from app.constants.enums import CandidateSource, BannedWordStatus
|
||||
from app.services.ai_engine.constants import QUALITY_PASS_SCORE
|
||||
|
||||
banned_rows = db.query(BannedWord).filter(
|
||||
BannedWord.workspace_id == workspace_id
|
||||
).all()
|
||||
banned_dicts = [{"word": b.word, "level": b.level, "replacement": b.replacement}
|
||||
for b in banned_rows]
|
||||
|
||||
# 查 key_id(只取 id,不解密,不违反基石B)
|
||||
key_row = db.query(UserApiKey).filter(
|
||||
UserApiKey.user_id == task.operator_id,
|
||||
UserApiKey.workspace_id == workspace_id,
|
||||
).first()
|
||||
key_id = key_row.id if key_row else None
|
||||
|
||||
t0 = time.monotonic()
|
||||
llm_success = True
|
||||
try:
|
||||
candidates_raw = asyncio.run(generate_text_variants(
|
||||
llm_client=clients,
|
||||
product=product_dict,
|
||||
count=task.text_count,
|
||||
banned_word_rows=banned_dicts,
|
||||
flywheel_context=flywheel_fragment,
|
||||
))
|
||||
except Exception as exc:
|
||||
llm_success = False
|
||||
logger.error("generate_text_variants 失败: %s", exc)
|
||||
candidates_raw = []
|
||||
latency_ms = int((time.monotonic() - t0) * 1000)
|
||||
|
||||
# 写 ai_call_logs(留痕,不含明文key)
|
||||
try:
|
||||
log = AiCallLog(
|
||||
workspace_id=workspace_id,
|
||||
user_id=task.operator_id,
|
||||
key_id=key_id,
|
||||
task_id=task.id,
|
||||
provider="apiports",
|
||||
model=clients._model,
|
||||
call_type="text",
|
||||
success=llm_success,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
db.add(log)
|
||||
db.flush()
|
||||
except Exception as log_exc:
|
||||
logger.warning("ai_call_logs 写入失败(非阻断): %s", log_exc)
|
||||
|
||||
# S1: 存库前过滤——只存合格文案(passed + score>=90 + 非hard_block)
|
||||
seq = seq_start
|
||||
saved_count = 0
|
||||
for i, c in enumerate(candidates_raw):
|
||||
score = c.get("score", 0)
|
||||
passed = c.get("passed", False)
|
||||
bw_status = c.get("banned_word_status", "pass")
|
||||
if not (passed and score >= QUALITY_PASS_SCORE and bw_status != "hard_block"):
|
||||
logger.info(
|
||||
"文案[%d] 过滤丢弃: passed=%s score=%s banned=%s",
|
||||
i, passed, score, bw_status,
|
||||
)
|
||||
continue
|
||||
|
||||
tc = TextCandidate(
|
||||
workspace_id=workspace_id,
|
||||
task_id=task.id,
|
||||
source=CandidateSource.AI,
|
||||
angle_label=c.get("angle_label") or c.get("angle", ""),
|
||||
content=json.dumps(c, ensure_ascii=False),
|
||||
score_json=json.dumps(c.get("score_detail", []), ensure_ascii=False),
|
||||
banned_word_status=BannedWordStatus(bw_status),
|
||||
)
|
||||
db.add(tc)
|
||||
db.flush()
|
||||
saved_count += 1
|
||||
seq += 1
|
||||
push_fn(task.id, workspace_id, "text_candidate", {
|
||||
"candidate_id": tc.id, "angle_label": tc.angle_label,
|
||||
"content": c.get("content", ""), "score": score,
|
||||
}, seq)
|
||||
seq += 1
|
||||
push_fn(task.id, workspace_id, "text_progress", {
|
||||
"done": saved_count, "total": task.text_count
|
||||
}, seq)
|
||||
|
||||
db.commit()
|
||||
|
||||
# S1: 合格数不足时标记需要后台补充
|
||||
needs_replenish = saved_count < task.text_count
|
||||
if needs_replenish:
|
||||
logger.warning(
|
||||
"文案合格数不足: task_id=%s 目标=%s 实得=%s,将后台异步补充",
|
||||
task.id, task.text_count, saved_count,
|
||||
)
|
||||
return candidates_raw, seq, needs_replenish
|
||||
|
||||
|
||||
def run_image_generation(db, clients, task, product_dict: dict,
|
||||
push_fn, workspace_id: int, seq_start: int,
|
||||
first_copy: dict, upload_base_path: str) -> int:
|
||||
"""
|
||||
Step6+7+8(image): 调 generate_storyboard_images → 后处理 → 存 ImageCandidate → 推 SSE。
|
||||
返回 next_seq。
|
||||
"""
|
||||
import time
|
||||
from app.services.ai_engine.image_gen import generate_storyboard_images
|
||||
from app.services.ai_engine.image_postprocessor import process_image
|
||||
from app.models.task import ImageCandidate
|
||||
from app.models.flywheel import AiCallLog
|
||||
from app.models.workspace import UserApiKey
|
||||
from app.constants.enums import ImageRole as IR
|
||||
|
||||
# 取 key_id(不解密,不记录明文 key)
|
||||
key_row = db.query(UserApiKey).filter(
|
||||
UserApiKey.user_id == task.operator_id,
|
||||
UserApiKey.workspace_id == workspace_id,
|
||||
).first()
|
||||
key_id = key_row.id if key_row else None
|
||||
|
||||
# TODO: 尺寸字段后续加产品级配置(products 表现无 aspect_ratio 字段)
|
||||
# 本轮固定 '3:4'=1024×1536,与 gpt-image-2 原生尺寸一致,免后处理二次拉伸
|
||||
aspect_ratio = "3:4"
|
||||
|
||||
# image_count=0 直接跳过(纯文案任务/测试),不空跑生图通道触发无谓失败日志。
|
||||
if not task.image_count or task.image_count <= 0:
|
||||
logger.info("image_count=0,跳过生图: task_id=%s", task.id)
|
||||
return seq_start
|
||||
|
||||
reference_images: list[bytes] = []
|
||||
_img_path = _resolve_image_path(product_dict.get("image_path", ""))
|
||||
if _img_path and os.path.isfile(_img_path):
|
||||
try:
|
||||
with open(_img_path, "rb") as _f:
|
||||
reference_images = [_f.read()]
|
||||
logger.info("产品参考图已加载:%s (%d bytes)", _img_path, len(reference_images[0]))
|
||||
except Exception as _e:
|
||||
logger.warning("产品参考图读取失败,退化为空列表:%s %s", _img_path, _e)
|
||||
else:
|
||||
logger.warning(
|
||||
"product.image_path 未设置或文件不存在(%r),生图将以无参考图模式运行,"
|
||||
"可能导致产品包装跑偏。", _img_path
|
||||
)
|
||||
|
||||
# 禁降级兜底:本次产品入镜但无参考图 → 硬失败,绝不降级纯文生图(建任务已拦一道,这是防绕过)
|
||||
if getattr(task, "need_product_image", True) and not reference_images:
|
||||
raise ValueError(
|
||||
"本次产品入镜(need_product_image=True)但未获取到产品参考图,"
|
||||
"拒绝降级纯文生图。请确认产品已上传参考图。"
|
||||
)
|
||||
|
||||
seq = seq_start
|
||||
# 3套正交叙事 A/B/C,每套各 image_count 张独立生图
|
||||
for strategy in ("A", "B", "C"):
|
||||
t0 = time.monotonic()
|
||||
img_success = True
|
||||
img_error_code = None
|
||||
try:
|
||||
image_results = asyncio.run(generate_storyboard_images(
|
||||
client=clients,
|
||||
note=first_copy,
|
||||
product=product_dict,
|
||||
image_count=task.image_count,
|
||||
reference_images=reference_images or None,
|
||||
strategy=strategy,
|
||||
))
|
||||
except Exception as exc:
|
||||
img_success = False
|
||||
img_error_code = type(exc).__name__
|
||||
logger.error("generate_storyboard_images 套%s 失败: %s", strategy, exc)
|
||||
image_results = []
|
||||
latency_ms = int((time.monotonic() - t0) * 1000)
|
||||
|
||||
fail_count = 0
|
||||
first_img_error: str | None = None
|
||||
for i, img_result in enumerate(image_results):
|
||||
if img_result.get("error"):
|
||||
fail_count += 1
|
||||
if first_img_error is None:
|
||||
first_img_error = str(img_result["error"])[:32]
|
||||
seq += 1
|
||||
push_fn(task.id, workspace_id, "batch_failed", {
|
||||
"batch": img_result["role"], "reason": img_result["error"],
|
||||
"strategy": strategy, "retryable": True,
|
||||
}, seq)
|
||||
continue
|
||||
|
||||
raw_bytes = img_result["image_bytes"]
|
||||
try:
|
||||
processed = process_image(raw_bytes, aspect_ratio=aspect_ratio, resample_strength=1)
|
||||
except Exception as e:
|
||||
logger.warning("图片后处理失败,使用原图: %s", e)
|
||||
processed = raw_bytes
|
||||
|
||||
img_dir = os.path.join(upload_base_path, str(workspace_id), str(task.id))
|
||||
os.makedirs(img_dir, exist_ok=True)
|
||||
filename = f"{strategy}_{i+1:02d}_{img_result['role']}.jpg"
|
||||
img_path = os.path.join(img_dir, filename)
|
||||
with open(img_path, "wb") as f:
|
||||
f.write(processed)
|
||||
|
||||
img_url = f"/uploads/{workspace_id}/{task.id}/{filename}"
|
||||
|
||||
role_enum = IR.MAIN
|
||||
try:
|
||||
role_enum = IR(img_result["role"])
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
ic = ImageCandidate(
|
||||
workspace_id=workspace_id,
|
||||
task_id=task.id,
|
||||
role=role_enum,
|
||||
url=img_url,
|
||||
seq=i + 1,
|
||||
strategy=strategy, # 写入 A/B/C(非 hardcode)
|
||||
)
|
||||
db.add(ic)
|
||||
db.flush()
|
||||
seq += 1
|
||||
push_fn(task.id, workspace_id, "image_candidate", {
|
||||
"candidate_id": ic.id, "strategy": strategy,
|
||||
"url": img_url, "role": img_result["role"],
|
||||
}, seq)
|
||||
seq += 1
|
||||
push_fn(task.id, workspace_id, "image_progress", {
|
||||
"done": i + 1, "total": task.image_count, "strategy": strategy,
|
||||
}, seq)
|
||||
|
||||
# 写 ai_call_logs(每套一条,失败不阻断)
|
||||
actual_provider = os.environ.get("IMAGE_PROVIDER_PRIMARY", "gpt")
|
||||
final_error_code = first_img_error or img_error_code
|
||||
try:
|
||||
img_log = AiCallLog(
|
||||
workspace_id=workspace_id,
|
||||
user_id=task.operator_id,
|
||||
key_id=key_id,
|
||||
task_id=task.id,
|
||||
provider=actual_provider,
|
||||
call_type="image",
|
||||
success=(img_success and fail_count == 0),
|
||||
latency_ms=latency_ms,
|
||||
error_code=final_error_code,
|
||||
)
|
||||
db.add(img_log)
|
||||
db.flush()
|
||||
except Exception as log_exc:
|
||||
logger.warning("ai_call_logs(image) 套%s 写入失败(非阻断): %s", strategy, log_exc)
|
||||
|
||||
db.commit()
|
||||
return seq
|
||||
93
backend/app/workers/pipeline_steps.py
Normal file
93
backend/app/workers/pipeline_steps.py
Normal file
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
app/workers/pipeline_steps.py — 生产链 Step1-4
|
||||
Step1: 查 DB(task/product)
|
||||
Step2: 查 key → Fernet 解密(局部变量,不传出,基石B)
|
||||
Step3: 构建 AI clients
|
||||
Step4: 推 task_started SSE + 飞轮上下文
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_task_and_product(db, task_id: int):
|
||||
"""Step1: 查任务 + 产品,失败返 None 或抛异常。"""
|
||||
from app.models.task import GenerationTask
|
||||
from app.models.product import Product
|
||||
|
||||
task = db.query(GenerationTask).filter(GenerationTask.id == task_id).first()
|
||||
if not task:
|
||||
logger.error("task_id=%s not found", task_id)
|
||||
return None, None
|
||||
|
||||
product = db.query(Product).filter(Product.id == task.product_id).first()
|
||||
if not product:
|
||||
raise ValueError(f"product_id={task.product_id} not found")
|
||||
return task, product
|
||||
|
||||
|
||||
def decrypt_user_key(db, operator_id: int, workspace_id: int) -> str:
|
||||
"""
|
||||
Step2: 查 key → Fernet 解密,返回 plain_key(只活在调用方局部变量)。
|
||||
绝不打印、不持久化 plain_key(基石B)。
|
||||
"""
|
||||
from app.models.workspace import UserApiKey
|
||||
from app.utils.fernet_utils import decrypt_key
|
||||
|
||||
api_key_row = db.query(UserApiKey).filter(
|
||||
UserApiKey.user_id == operator_id,
|
||||
UserApiKey.workspace_id == workspace_id,
|
||||
UserApiKey.provider.in_(["openai", "apiports"]), # G6坑修复:接受主备通道名
|
||||
).first()
|
||||
if not api_key_row:
|
||||
raise ValueError("用户未配置 API Key,请先录入")
|
||||
return decrypt_key(api_key_row.encrypted_key)
|
||||
|
||||
|
||||
def build_clients_and_clear_key(plain_key: str):
|
||||
"""
|
||||
Step3: 构建 AIClients,plain_key 传入后立即由调用方置 None。
|
||||
返回 clients 对象。
|
||||
"""
|
||||
from app.services.ai_engine.gemini_factory import build_ai_clients
|
||||
return build_ai_clients(plain_key)
|
||||
|
||||
|
||||
def build_product_dict(product) -> dict:
|
||||
"""把 ORM product 转成 AI 引擎所需的 dict(不含任何 key)。"""
|
||||
return {
|
||||
"name": product.name,
|
||||
"category": product.category or "通用好物",
|
||||
"selling_points": json.loads(product.selling_points or "[]"),
|
||||
"style_tone": product.style_tone or "素人分享风",
|
||||
"text_angles": json.loads(product.text_angles or "[]"),
|
||||
"custom_prompt": product.custom_prompt or "",
|
||||
"brand_keyword": product.brand_keyword or "", # S3: 品牌词透传进生成prompt(每条植入)
|
||||
"target_audience": product.target_audience or "", # 012: 人群透传进storyboard/文案prompt
|
||||
"image_path": product.image_path or "", # 产品参考图路径(前端上传后填入)
|
||||
}
|
||||
|
||||
|
||||
def load_flywheel_context(db, workspace_id: int, product_id: int, product_dict: dict) -> tuple[str, dict]:
|
||||
"""
|
||||
查最近50条飞轮事件,聚合偏好上下文。
|
||||
返回 (prompt_fragment, full_ctx)。
|
||||
"""
|
||||
from app.models.flywheel import PreferenceEvent
|
||||
from app.services.ai_engine.preference_aggregator import aggregate_preference_context
|
||||
|
||||
recent = db.query(PreferenceEvent).filter(
|
||||
PreferenceEvent.workspace_id == workspace_id,
|
||||
PreferenceEvent.product_id == product_id,
|
||||
).order_by(PreferenceEvent.id.desc()).limit(50).all()
|
||||
|
||||
events_dicts = [
|
||||
{"signal_type": e.signal_type, "workspace_id": e.workspace_id,
|
||||
"product_id": e.product_id, "angle_label": e.angle_label or "",
|
||||
"signal_weight": e.signal_weight, "reason": e.reason or ""}
|
||||
for e in recent
|
||||
]
|
||||
ctx = aggregate_preference_context(events_dicts, product_dict, workspace_id, product_id)
|
||||
return ctx.get("prompt_fragment", ""), ctx
|
||||
100
backend/app/workers/replenish_task.py
Normal file
100
backend/app/workers/replenish_task.py
Normal file
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
app/workers/replenish_task.py — 文案后台补充任务(S1: 先展示后台补铁律)
|
||||
|
||||
合格文案(passed且score>=90且非hard_block)不足用户目标条数时,
|
||||
run_generation_pipeline 已先展示合格的; 此任务异步补齐到目标数或达上限。
|
||||
只接收 task_id(基石B)。复用 pipeline_io.run_text_generation 的生成+过滤。
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from app.workers.celery_app import celery_app
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 补充上限:避免合格率低时无限补。最多补 MAX_REPLENISH_ROUNDS 轮。
|
||||
MAX_REPLENISH_ROUNDS = 3
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.workers.tasks.replenish_text_candidates",
|
||||
max_retries=2,
|
||||
default_retry_delay=30,
|
||||
queue="generation",
|
||||
)
|
||||
def replenish_text_candidates(self, task_id: int) -> dict:
|
||||
"""后台补充合格文案到目标条数(或达补充上限)。
|
||||
|
||||
每轮调用 run_text_generation 续生成,过滤后累加入库;
|
||||
已合格数 >= text_count 即停; 达 MAX_REPLENISH_ROUNDS 仍不足也停(不卡用户)。
|
||||
"""
|
||||
from app.workers.tasks import _get_db, _push_event_sync
|
||||
from app.workers.pipeline_io import run_text_generation
|
||||
from app.workers.pipeline_steps import (
|
||||
load_task_and_product,
|
||||
decrypt_user_key,
|
||||
build_clients_and_clear_key,
|
||||
build_product_dict,
|
||||
load_flywheel_context,
|
||||
)
|
||||
from app.models.task import TextCandidate
|
||||
|
||||
db = _get_db()
|
||||
try:
|
||||
task, product = load_task_and_product(db, task_id)
|
||||
if not task:
|
||||
logger.warning("补充任务找不到 task_id=%s", task_id)
|
||||
return {"task_id": task_id, "replenished": 0, "reason": "task_not_found"}
|
||||
|
||||
# 已合格数(按入库的合格候选计)
|
||||
existing = db.query(TextCandidate).filter(
|
||||
TextCandidate.task_id == task_id
|
||||
).count()
|
||||
target = task.text_count
|
||||
if existing >= target:
|
||||
return {"task_id": task_id, "replenished": 0, "reason": "already_enough"}
|
||||
|
||||
workspace_id = task.workspace_id
|
||||
plain_key = decrypt_user_key(db, task.operator_id, workspace_id)
|
||||
clients = build_clients_and_clear_key(plain_key)
|
||||
plain_key = None # 用完即清除,基石B
|
||||
product_dict = build_product_dict(product)
|
||||
flywheel_fragment, _ = load_flywheel_context(
|
||||
db, workspace_id, task.product_id, product_dict
|
||||
)
|
||||
|
||||
seq = 9000 # 补充事件 seq 段,避开主流程
|
||||
rounds = 0
|
||||
# 真实入库合格数以 DB count 为准,绝不用 len(raw)(那是原始生成数含被过滤的低分条)。
|
||||
# task47 实测踩坑:len(raw)=2 但 saved=0(都没过90),误判达标提前停,库里仍缺。
|
||||
current = existing
|
||||
while current < target and rounds < MAX_REPLENISH_ROUNDS:
|
||||
rounds += 1
|
||||
run_text_generation(
|
||||
db, clients, task, product_dict, flywheel_fragment,
|
||||
_push_event_sync, workspace_id, seq,
|
||||
)
|
||||
# run_text_generation 内部已把本轮合格候选写库,这里重查真实库内条数
|
||||
db.expire_all()
|
||||
current = db.query(TextCandidate).filter(
|
||||
TextCandidate.task_id == task_id
|
||||
).count()
|
||||
seq += 100
|
||||
logger.info(
|
||||
"补充轮 %d: task_id=%s 库内合格=%d/%d",
|
||||
rounds, task_id, current, target,
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"replenished": current - existing,
|
||||
"rounds": rounds,
|
||||
"final": current,
|
||||
"target": target,
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.error("replenish_text_candidates failed: task_id=%s err=%s", task_id, exc)
|
||||
raise self.retry(exc=exc)
|
||||
finally:
|
||||
db.close()
|
||||
155
backend/app/workers/tasks.py
Normal file
155
backend/app/workers/tasks.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
app/workers/tasks.py — Celery 任务入口(编排层)
|
||||
只接收 task_id(基石B)。具体步骤在 pipeline_steps / pipeline_io。
|
||||
打包任务在 packaging_task.py,此处 re-export 保持旧引用不变。
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from app.workers.celery_app import celery_app
|
||||
# re-export 保持旧代码 `from app.workers.tasks import build_delivery_package` 不变
|
||||
from app.workers.packaging_task import build_delivery_package # noqa: F401
|
||||
from app.workers.replenish_task import replenish_text_candidates # noqa: F401
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_db():
|
||||
"""获取同步 DB Session(Celery worker 环境用同步)"""
|
||||
from app.core.database import SessionLocal
|
||||
return SessionLocal()
|
||||
|
||||
|
||||
def _push_event_sync(task_id: int, workspace_id: int, event: str, data: dict, seq: int):
|
||||
"""同步推 SSE 事件(worker 内部用)"""
|
||||
import json as _json
|
||||
import redis as redis_lib
|
||||
from app.core.config import get_settings
|
||||
s = get_settings()
|
||||
r = redis_lib.from_url(s.REDIS_URL, decode_responses=True)
|
||||
hist_key = f"sse:task:{task_id}:events"
|
||||
channel = f"sse:task:{task_id}"
|
||||
record = _json.dumps({"event": event, "data": data, "seq": seq,
|
||||
"workspace_id": workspace_id}, ensure_ascii=False)
|
||||
r.rpush(hist_key, record)
|
||||
r.expire(hist_key, 3600)
|
||||
r.publish(channel, record)
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.workers.tasks.run_generation_pipeline",
|
||||
max_retries=3,
|
||||
default_retry_delay=30,
|
||||
queue="generation",
|
||||
)
|
||||
def run_generation_pipeline(self, task_id: int) -> dict:
|
||||
"""
|
||||
生产链主任务。只接收 task_id,绝不接收 key。
|
||||
子步骤委托给 pipeline_steps(查库/解密/飞轮上下文)
|
||||
和 pipeline_io(文案/图片生成+存储)。
|
||||
"""
|
||||
logger.info("run_generation_pipeline start: task_id=%s", task_id)
|
||||
db = _get_db()
|
||||
seq = 0
|
||||
workspace_id = 0 # G3坑修复:确保 except 里能用真实 workspace_id
|
||||
|
||||
try:
|
||||
from app.constants.enums import TaskStatus
|
||||
from app.workers.pipeline_steps import (
|
||||
load_task_and_product,
|
||||
decrypt_user_key,
|
||||
build_clients_and_clear_key,
|
||||
build_product_dict,
|
||||
load_flywheel_context,
|
||||
)
|
||||
from app.workers.pipeline_io import run_text_generation, run_image_generation
|
||||
from app.core.config import get_settings
|
||||
|
||||
# Step1: 查任务 + 产品
|
||||
task, product = load_task_and_product(db, task_id)
|
||||
if not task:
|
||||
return {"task_id": task_id, "status": "not_found"}
|
||||
|
||||
workspace_id = task.workspace_id
|
||||
|
||||
# Step2: Fernet 解密(局部变量,不传出)
|
||||
plain_key = decrypt_user_key(db, task.operator_id, workspace_id)
|
||||
|
||||
# Step3: 构建 AIClients
|
||||
clients = build_clients_and_clear_key(plain_key)
|
||||
plain_key = None # 用完即清除,基石B
|
||||
|
||||
task.status = TaskStatus.GENERATING
|
||||
db.commit()
|
||||
|
||||
# Step4: 推 task_started + 飞轮上下文
|
||||
seq += 1
|
||||
_push_event_sync(task_id, workspace_id, "task_started", {
|
||||
"task_id": task_id, "total_text": task.text_count, "total_image": task.image_count,
|
||||
}, seq)
|
||||
|
||||
product_dict = build_product_dict(product)
|
||||
flywheel_fragment, flywheel_ctx = load_flywheel_context(db, workspace_id, task.product_id, product_dict)
|
||||
|
||||
if flywheel_fragment:
|
||||
seq += 1
|
||||
_push_event_sync(task_id, workspace_id, "flywheel_injected", {
|
||||
"recent_preference": flywheel_ctx.get("recent_preference", ""),
|
||||
"reject_reasons": flywheel_ctx.get("reject_reasons", []),
|
||||
}, seq)
|
||||
|
||||
# Step5: 文案生成(S1: 返回 needs_replenish=合格数<目标数,触发后台补充)
|
||||
candidates_raw, seq, needs_replenish = run_text_generation(
|
||||
db, clients, task, product_dict, flywheel_fragment,
|
||||
_push_event_sync, workspace_id, seq,
|
||||
)
|
||||
if needs_replenish:
|
||||
# 合格文案不足用户目标条数:先展示已合格的,后台异步补充(先展示后台补铁律)
|
||||
# 注:candidates_raw 是本轮原始生成数(含被过滤的低分条),非合格入库数;
|
||||
# 真实合格数以库内 saved_count 为准,补充子任务会按库内实际缺口补到够。
|
||||
logger.warning(
|
||||
"文案合格数不足将后台补充: task_id=%s 本轮生成=%d 目标=%d",
|
||||
task_id, len(candidates_raw), task.text_count,
|
||||
)
|
||||
try:
|
||||
replenish_text_candidates.delay(task_id)
|
||||
except Exception as _re:
|
||||
logger.error("触发后台补充失败: task_id=%s err=%s", task_id, _re)
|
||||
|
||||
# Step6+7+8: 图片生成 + 后处理 + 存盘
|
||||
first_copy = candidates_raw[0] if candidates_raw else {}
|
||||
settings = get_settings()
|
||||
seq = run_image_generation(
|
||||
db, clients, task, product_dict,
|
||||
_push_event_sync, workspace_id, seq,
|
||||
first_copy, settings.UPLOAD_BASE_PATH,
|
||||
)
|
||||
|
||||
# 最终状态 + task_done
|
||||
task.status = TaskStatus.PENDING_SELECTION
|
||||
db.commit()
|
||||
|
||||
seq += 1
|
||||
_push_event_sync(task_id, workspace_id, "task_done", {
|
||||
"task_id": task_id, "status": "pending_selection"
|
||||
}, seq)
|
||||
|
||||
logger.info("run_generation_pipeline done: task_id=%s", task_id)
|
||||
return {"task_id": task_id, "status": "pending_selection"}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("run_generation_pipeline failed: task_id=%s err=%s", task_id, exc)
|
||||
try:
|
||||
from app.models.task import GenerationTask as GT
|
||||
from app.constants.enums import TaskStatus as TS
|
||||
t = db.query(GT).filter(GT.id == task_id).first()
|
||||
if t:
|
||||
t.status = TS.PENDING
|
||||
db.commit()
|
||||
except Exception:
|
||||
pass
|
||||
_push_event_sync(task_id, workspace_id, "error", {"code": 50001, "message": str(exc)}, seq + 1)
|
||||
raise self.retry(exc=exc)
|
||||
finally:
|
||||
db.close()
|
||||
Reference in New Issue
Block a user