diff --git a/backend/packages/ai_engine/pyproject.toml b/backend/packages/ai_engine/pyproject.toml index bb279425..84492ec8 100644 --- a/backend/packages/ai_engine/pyproject.toml +++ b/backend/packages/ai_engine/pyproject.toml @@ -10,6 +10,8 @@ dependencies = [ "langchain-core>=0.3", "pillow>=10.4", "numpy>=1.26", + "imageio>=2.36", + "av>=14.0", # imageio pyav 后端(视频抽帧) # "rembg", # 抠图(按需启用) ] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/_imgio.py b/backend/packages/ai_engine/src/windup_ai_engine/_imgio.py new file mode 100644 index 00000000..acec47d6 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/_imgio.py @@ -0,0 +1,26 @@ +"""PNG bytes ↔ PIL 的唯一转换口。 + +管线内部按 ``PIL.Image`` 处理,跨模块边界(strategy → generator → ports 出参)按 PNG +bytes 传递。这对转换此前在 ``strategy.concrete`` 与 ``impl.character_generator`` 各写 +了一份,收成一处 —— 编码参数(如是否强制 RGBA)一旦分叉,会在"某些帧丢了 alpha"这类 +只在画面上体现、不报错的地方出问题。 +""" +from __future__ import annotations + +import io + +from PIL import Image + +__all__ = ["to_png", "from_png"] + + +def to_png(img: Image.Image) -> bytes: + """PIL → PNG bytes。统一转 RGBA:下游脚线对齐靠 alpha 求包围盒。""" + buf = io.BytesIO() + img.convert("RGBA").save(buf, "PNG") + return buf.getvalue() + + +def from_png(png: bytes) -> Image.Image: + """PNG bytes → RGBA 图。""" + return Image.open(io.BytesIO(png)).convert("RGBA") diff --git a/backend/packages/ai_engine/src/windup_ai_engine/_subject.py b/backend/packages/ai_engine/src/windup_ai_engine/_subject.py new file mode 100644 index 00000000..bf83ab29 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/_subject.py @@ -0,0 +1,68 @@ +"""「哪些像素是主体」的唯一定义(母版预检 / 脚线 / 补边背景色共用)。 + +此前这套判据有两份:``master_prep._bg_color`` 取四角中位色补边, +``slicing.oneshot._subject_rows`` 用同一套四角中位色 + 容差找脚线。入口预检 +(:mod:`.master_check`)必须与下游用**同一个**主体定义 —— 判据一旦分叉就会出现 +"预检说有主体、下游找不到主体"这种只在画面上体现、不报错的分歧,和 +:mod:`._imgio` / :mod:`.slicing._frames` 当初被收拢是同一个理由。 + +判据本身:有真 alpha(存在低于阈值的像素)就用 alpha;整幅不透明(原始视频帧 / +RGB 母版)则按四角中位背景色的差值。**这是颜色启发式,不是抠图模型** —— +背景带渐变、或角色与背景同色时判不准,见 :func:`subject_mask`。 +""" +from __future__ import annotations + +import numpy as np +from PIL import Image + +__all__ = ["bg_color", "subject_bbox", "subject_mask"] + +ALPHA_THR = 128 # alpha 高于此值算不透明(与 postprocess.pack 求包围盒的口径一致) +BG_TOL = 60 # 与背景色的 RGB 绝对差之和,超过才算主体 + + +def _bg_median(rgb: np.ndarray) -> np.ndarray: + """四角中位色(float)。母版 / 视频帧通常是纯色底,四角取中位比取均值抗单角污染。""" + corners = np.stack([rgb[0, 0], rgb[0, -1], rgb[-1, 0], rgb[-1, -1]]) + return np.median(corners, axis=0) + + +def bg_color(img: Image.Image) -> tuple[int, int, int]: + """背景色(取整),给补边用。""" + rgb = np.asarray(img.convert("RGB")) + return tuple(int(v) for v in _bg_median(rgb)) + + +def subject_mask( + img: Image.Image, alpha_thr: int = ALPHA_THR, bg_tol: int = BG_TOL +) -> np.ndarray: + """主体像素的二维布尔掩码。 + + 必须兼容**不透明**输入:抽帧阶段拿到的是原始视频帧,还没抠图,只看 alpha 会把 + 整幅当主体、脚线恒定,腾空判据立刻误判"已落地"(实测踩过,跳跃被裁在起跳前)。 + + 判不准的已知情形(调用方别当成抠图):背景有渐变 → 整幅都超容差,掩码≈全 True; + 角色主色与背景色接近 → 那部分身体被判成背景。要真分割请走 MatteProvider。 + """ + arr = np.asarray(img.convert("RGBA")) + alpha = arr[:, :, 3] + if not alpha.min() > alpha_thr: # 存在透明像素 = 有真 alpha,直接用 + return alpha > alpha_thr + rgb = arr[:, :, :3].astype(np.int16) + return np.abs(rgb - _bg_median(rgb)).sum(axis=2) > bg_tol + + +def subject_bbox( + img: Image.Image, alpha_thr: int = ALPHA_THR, bg_tol: int = BG_TOL +) -> tuple[tuple[int, int, int, int], int] | None: + """主体包围盒 ``(x0, y0, x1, y1)``(半开,同 PIL crop)+ 主体像素数;无主体返回 None。 + + 包围盒与像素数一起返回:两者判的不是同一件事 —— 包围盒管"主体有多大", + 像素数管"包围盒里是不是真有东西"(散落的几粒噪点能把包围盒撑满整幅)。 + """ + m = subject_mask(img, alpha_thr, bg_tol) + ys, xs = np.where(m) + if not len(ys): + return None + box = (int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1) + return box, int(m.sum()) diff --git a/backend/packages/ai_engine/src/windup_ai_engine/impl/__init__.py b/backend/packages/ai_engine/src/windup_ai_engine/impl/__init__.py new file mode 100644 index 00000000..456b868a --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/impl/__init__.py @@ -0,0 +1,5 @@ +"""impl:CharacterGeneratorPort 的装配实现(串联 strategy + 最后一公里)。""" + +from .character_generator import CharacterGenerator + +__all__ = ["CharacterGenerator"] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/impl/character_generator.py b/backend/packages/ai_engine/src/windup_ai_engine/impl/character_generator.py new file mode 100644 index 00000000..1a4458d7 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/impl/character_generator.py @@ -0,0 +1,168 @@ +"""CharacterGenerator —— 装配 strategy + 最后一公里,串起整条生产线(架构串联点)。 + +这是 CharacterGeneratorPort 的实现;server 经 port 调它、不碰这里。 +串联:母版预检(可拒绝)→ 选路线(ROUTE_MATRIX)→ strategy.derive 出帧 → +最后一公里(脚线对齐)→ 量交付成色 → GeneratedAction。 + +两头各有一道闸,方向相反:进门那道(master_check)在**花钱之前**挡住不可能生成好的 +输入;出门那几道(空帧 / 帧数 / 成色)在钱已经花完之后,挡住"看起来成功的错产物"。 + +MVP 边界(与作者对齐):**只出帧 bytes + 逐帧时长**,不打包 sprite sheet、不落存储—— +上传对象存储、写 character_data、拼图集/多格式导出由 server / export 侧做(#22)。 +""" +from __future__ import annotations + +import numpy as np +from PIL import Image + +from windup_common.models import ActionSpec, CharacterCard, GenRoute + +from windup_ai_engine._imgio import from_png as _img +from windup_ai_engine._imgio import to_png as _png +from windup_ai_engine.master_check import check_master +from windup_ai_engine.ports import ( + ActionQuality, + CharacterGeneratorPort, + GeneratedAction, + ProgressPort, +) +from windup_ai_engine.postprocess import align_bottom_center, frame_durations +from windup_ai_engine.slicing import dead_frame_indices, loop_seam, motion_scale +from windup_ai_engine.strategy.base import ( + CYCLIC_ACTIONS, + ROUTE_MATRIX, + DerivationStrategy, +) + + + + +class CharacterGenerator(CharacterGeneratorPort): + """由 bootstrap 注入 {GenRoute: DerivationStrategy} 装配表。""" + + def __init__(self, strategies: dict[GenRoute, DerivationStrategy]) -> None: + self._by_route = strategies + + def generate( + self, + card: CharacterCard, + action: ActionSpec, + master: bytes, + progress: ProgressPort, + canvas: tuple[int, int] | None = None, + ) -> GeneratedAction: + # ① 入口预检 —— 唯一一道在**花钱之前**的闸,故排在选路线之前。 + # 之前这里什么都不判:一张"人物在画板前作画"的图请求 walk,全程无一处报错, + # 16 帧构图完整的错角色出完、钱花完(2026-08-07 实测)。预检拦不住"内容画错" + # (那要视觉模型),但坏图 / 空图 / 极端比例这几类不必等到出帧才发现。 + # 预检与出帧必须用**同一个** canvas:比例上限是由交付画布几何推出来的, + # 传一个、出另一个就等于预检按方形判、出帧按非方出(见 master_check)。 + facts = check_master(master, canvas) + progress.step("precheck", 0, 4, facts.note()) + + # ② 选路线(架构决策矩阵)。装配表里没有 = 该路线未实现,在边界上炸, + # 不要让"看着成功、内容是空"的结果流到 server 去落库。 + route = ROUTE_MATRIX[action.action] + # .value 而不是枚举本身:Python 3.11+ 的 str-mixin 枚举 __format__ 会给出 + # "ActionType.WALK",这串字最终是用户看到的进度文案(3.12.13 实测)。 + progress.step("route", 1, 4, f"{action.action.value} → {route.value}") + strategy = self._by_route.get(route) + if strategy is None: + raise NotImplementedError( + f"动作 {action.action.value} 分流到 {route.value}，但未注入该路线的 strategy。" + f"已装配：{sorted(r.value for r in self._by_route)}。" + ) + + # ③ 生成帧(交给 strategy —— 串联) + frames = strategy.derive(card, action, master, progress) + + # ③.5 帧数必须与契约相符。A2 把 n_frames 从 len(poses) 的推导值改成调用方直接声明的 + # 承诺,而抽帧那两个函数都会**静默少给**:slicing.pick_cycle / pick_oneshot 在 + # `len(dense) <= n`(或动作区间比 n 短)时 return frames/span,长度不足且不报错 + # (2026-08-08 读码复核)。少给的后果不是崩溃而是"短一截的动作":时长表由 + # frame_durations(…, len(frames)) 现算,长度自洽,server 看不出异常,用户拿到 + # 一段步子没走完的循环。故在此对账 —— 钱已经花了,但至少不让错产物流下去。 + # 放在 generator 而不是某个 strategy 里:这样将来任何新路线都受同一条约束。 + if len(frames) != action.n_frames: + raise ValueError( + f"{route.value} 要 {action.n_frames} 帧,实际产出 {len(frames)} 帧。" + "抽帧源帧数不足(i2v 视频太短 / 动作区间过窄)时会静默少给," + "请调小 n_frames 或加长视频。" + ) + + # ④ 最后一公里:脚线对齐成原地序列帧(直接对齐到调用方要的画布尺寸) + aligned = self._lastmile(frames, progress, canvas) + + # ⑤ 量交付成色。在**对齐之后**量,量的是用户真正会看到的那组帧:抠图 / 像素化 / + # 对齐都会改像素,在中间任何一步量出来的数都描述不了交付物。 + quality = self._assess(aligned, action) + + # ⑥ 出参:帧 + 逐帧时长 + 成色(上传 / 落库在 server 侧) + progress.step( + "package", 3, 4, + f"{len(aligned)} 帧 + 逐帧时长(动量 {quality.motion_scale:.2f}," + f"死帧 {len(quality.dead_frames)}/{len(aligned)})", + ) + return GeneratedAction( + frames=[_png(im) for im in aligned], + durations=frame_durations(action.action.value, len(aligned)), + quality=quality, + ) + + def _assess(self, frames: list[Image.Image], action: ActionSpec) -> ActionQuality: + """量交付帧的成色。这些数只上报、**不改动产物**,也不在此处代替调用方做判决。 + + 为什么不在这里直接对着阈值抛错:交付 / 重试 / 让用户换母版是产品决策,阈值该由 + server 按场景定;而且到这一步钱已经花完,引擎单方面丢弃产物只是把损失变成两份。 + 引擎负责"如实报数",不负责"替上层决定这次算不算数"。 + + ``loop_seam`` 只对循环类动作量:一次性动作(jump/attack)首尾姿态本就不同, + 给它算一个"接缝"再交出去,等于发一个必然难看的数让上层照着做错误决定。 + """ + return ActionQuality( + motion_scale=motion_scale(frames), + dead_frames=dead_frame_indices(frames), + loop_seam=loop_seam(frames) if action.action in CYCLIC_ACTIONS else None, + ) + + def _lastmile( + self, + frames: list[bytes], + progress: ProgressPort, + canvas: tuple[int, int] | None = None, + ) -> list[Image.Image]: + """脚线对齐:把各帧对齐成原地序列帧(消除逐帧画布漂移,Issue #21)。 + + 返回 PIL 而不是 PNG bytes:紧接着的成色测量要按图看帧,再编码回 PNG 只为了 + 让上一句话好听、下一句话又得解码回来。编码统一在 ``generate`` 出参那一步做。 + + 位移轨道(root_motion)MVP 先不做(见 #63 / character_data.frames 暂无该字段): + 序列帧保持原地即可,位移留给后续 export / playtest 阶段再算。 + + ``canvas`` 给定时直接对齐到该尺寸,而不是恒出 256 再让上层缩。上层那次缩放 + (``Image.thumbnail`` 补边)**只缩不放**:项目要 512 时 256 的帧不会被放大,而是 + 原尺寸居中贴进 512 画布,于是这里刚对齐好的脚线 0.92 被挪到 0.709(2026-08-11 + 实测),角色不站在地上、跨动作对齐也失效。在这里一次出到位就没有那一步了。 + """ + progress.step("lastmile", 2, 4, "脚线对齐(原地)") + # 空帧不再静默跳过:未实现的路线现在在 strategy / 装配表处就抛错(见 generate), + # 走到这里还有空帧说明 provider 或抠图吐了坏数据,同样要炸而不是原样放行。 + if not frames: + raise ValueError("strategy 未产出任何帧") + bad = [i for i, f in enumerate(frames) if not f] + if bad: + raise ValueError(f"strategy 产出了 {len(bad)}/{len(frames)} 个空帧，索引 {bad[:8]}") + imgs = [_img(f) for f in frames] + # 参考姿态高 = 各帧包围盒高的中位数:比"最高帧"稳(不被举过头顶的武器带偏), + # 各动作都以自身中位姿态定标,本体尺寸跨动作一致。 + hs = [] + for im in imgs: + ys, _ = np.where(np.asarray(im)[:, :, 3] > 128) + if len(ys): + hs.append(float(ys.max() - ys.min())) + # TODO(dev, #21): tail_match 循环闭合(净位移动作先锚点再匹配帧) + ref = float(np.median(hs)) if hs else None + if canvas is None: + return align_bottom_center(imgs, ref_height=ref) + cw, ch = canvas + return align_bottom_center(imgs, cell=cw, cell_h=ch, ref_height=ref) diff --git a/backend/packages/ai_engine/src/windup_ai_engine/master_check.py b/backend/packages/ai_engine/src/windup_ai_engine/master_check.py new file mode 100644 index 00000000..8b3376ea --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/master_check.py @@ -0,0 +1,164 @@ +"""母版可生成性预检 —— 入口处**允许拒绝**的那道闸,在花钱之前。 + +为什么有这个模块:ai_engine 此前所有 ``raise`` 都在输出侧,``generate(card, action, +master, progress)`` 对 ``master`` 一个前置判定都没有。2026-08-07 实测:喂一张"人物在 +画板前作画"的图请求 walk,全程无一处报错,最终产出 16 帧构图完整的序列帧,画面是个 +不会走路的错角色 —— 钱已花完才发现。 + +**本层判什么(三条,全部本地零成本、可复现):** + ① 能否解码 —— 坏 bytes / 截断文件不必等 i2v 跑完再发现; + ② 有没有可动的主体 —— 全透明 / 全同色 = 画面里没有东西可动; + ③ 主体宽高比下游装不装得下 —— 见 :data:`REJECT_ASPECT`。 + +**本层不判什么、为什么 —— 别把下面这些当成已经守住了:** + - **画的是不是一个角色、是不是该动作要的姿态**(walk 要侧向、attack 要蓄力,见 + :data:`master_prep.MASTER_POSES`)。需要视觉模型读画面语义,本层只有 numpy。 + **开头那张"人物在画板前作画"的图,本预检拦不住**:它能解码、有主体、比例正常。 + 本层挡的是它的近邻(空图 / 坏图 / 极端比例),挡不住"内容画错"。要真正堵住这个, + 得在预检里接一次廉价的视觉判定(便宜的 VLM 问一句"这是不是一个可行走的角色、 + 朝向是不是侧面"),那是另一件事、要另外的实测与预算。 + - **朝向与 ``ActionSpec.facing`` 是否一致** —— 同上,需要视觉模型。 + - **背景干不干净到能抠图** —— 抠图是 ``MatteProvider``(rembg/u2net)的事;本层的 + 四角中位色启发式判不出"这块背景 rembg 能不能抠掉"。 + - **分辨率下限** —— 故意不判。i2v 供应商对首帧分辨率的真实下限我没有实测数据, + 拍一个阈值就是拿没验证的判据挡掉用户的钱。:data:`MIN_SUBJECT_SIDE` 只挡退化端 + (小到与噪点无从区分),不是画质阈值。 + +纯 PIL / numpy,零 API,不联网。 +""" +from __future__ import annotations + +import io +from dataclasses import dataclass + +from PIL import Image, UnidentifiedImageError + +from windup_ai_engine._subject import subject_bbox +from windup_ai_engine.ports import MasterRejectCode, MasterRejected +from windup_ai_engine.postprocess.pack import FILL_H, FILL_W + +__all__ = ["MIN_SUBJECT_AREA_RATIO", "MIN_SUBJECT_SIDE", "REJECT_ASPECT", + "MasterFacts", "check_master", "reject_aspect_for"] + +# 主体宽高比上限。**由交付画布的几何推出,不是拍的**:align_bottom_center 按高定标 +# (cell*FILL_H);主体 w/h 超过 FILL_W/FILL_H(≈1.55)后宽度兜底接管,交付主体高度 +# 退化成 cell*FILL_W/(w/h)。取"退化到目标高度的一半"为界: +# FILL_W / R < FILL_H / 2 ⇒ R > 2*FILL_W/FILL_H ≈ 3.1 +# 再宽就不是"缩小了一点",是把角色压成一条。pack.py 记的实测(2026-08-05):w/h=1.78 +# 的狐狸母版丢 27px、w/h=2.0 只剩 79.9% 内容 —— 那还在兜底能救的区间内(交付变矮), +# 3.1 以上则是"硬缩到没法看"。与其硬缩出一个能落库的错产物,不如在花钱前退回去。 +REJECT_ASPECT = 2 * FILL_W / FILL_H + + +def reject_aspect_for(canvas: tuple[int, int] | None) -> float: + """给定交付画布下的实际比例上限。方形画布(或不指定)即 :data:`REJECT_ASPECT`。 + + 上面那条推导默认画布是方形 —— ``FILL_W`` 与 ``FILL_H`` 是同一条边长的两个比例。 + 画布可以非方之后这个前提就不成立了:宽度兜底是 ``cw*FILL_W/主体宽``、高度目标是 + ``ch*FILL_H/主体高``,同一条推导做下来是 + + R = 2 * (cw/ch) * FILL_W / FILL_H = REJECT_ASPECT * (cw/ch) + + 即窄高画布(cw str: + """给 ProgressPort 的一行摘要(会经 server 变成用户看到的进度文案)。""" + w, h = self.size + x0, y0, x1, y1 = self.subject_box + return (f"母版 {w}×{h},主体 {x1 - x0}×{y1 - y0}" + f"(w/h {self.subject_ratio:.2f},占幅 {self.subject_area_ratio:.1%})") + + +def _decode(master: bytes) -> Image.Image: + """解码母版;坏 bytes 直接拒。 + + 必须 ``load()`` 强制解完:``Image.open`` 只读文件头,截断的 PNG 在 open 处不报错, + 要到下游某个 ``convert`` / ``np.asarray`` 才炸 —— 那时 i2v 的钱已经花了。 + """ + if not master: + raise MasterRejected(MasterRejectCode.UNDECODABLE, "母版为空 bytes") + try: + img = Image.open(io.BytesIO(master)) + img.load() + return img.convert("RGBA") + except (UnidentifiedImageError, OSError, ValueError) as exc: + raise MasterRejected( + MasterRejectCode.UNDECODABLE, f"解不开这张图({type(exc).__name__}: {exc})" + ) from exc + + +def check_master(master: bytes, canvas: tuple[int, int] | None = None) -> MasterFacts: + """母版可生成性预检。通过返回量到的形态,不通过抛 :class:`MasterRejected`。 + + 只看母版本身,不看 ``ActionSpec``:三条判据都是"下游画布装不装得下 / 有没有东西可 + 动",与动作类型无关。动作相关的母版要求(侧向 / 蓄力姿态)本层判不了,见模块 docstring。 + + ``canvas``:交付画布 ``(宽, 高)``。只影响比例上限 —— 见 :func:`reject_aspect_for`。 + 不给即按方形判(与加这个入参之前完全一致)。**必须与出帧用的是同一个 canvas**, + 否则就成了"预检按一套几何判、出帧按另一套出"。 + """ + img = _decode(master) + w, h = img.size + found = subject_bbox(img) + if found is None: + raise MasterRejected( + MasterRejectCode.NO_SUBJECT, + f"{w}×{h} 的图里找不到主体(全透明或全同色),没有可动的东西", + ) + box, pixels = found + bw, bh = box[2] - box[0], box[3] - box[1] + facts = MasterFacts( + size=(w, h), + subject_box=box, + subject_ratio=bw / bh, + subject_area_ratio=pixels / max(1, w * h), + ) + if min(bw, bh) < MIN_SUBJECT_SIDE: + raise MasterRejected( + MasterRejectCode.SUBJECT_TOO_SMALL, + f"主体包围盒只有 {bw}×{bh}px(下限 {MIN_SUBJECT_SIDE}px)," + "与一粒噪点/水印无从区分", + ) + if facts.subject_area_ratio < MIN_SUBJECT_AREA_RATIO: + raise MasterRejected( + MasterRejectCode.SUBJECT_TOO_SMALL, + f"主体只占画幅 {facts.subject_area_ratio:.4%}" + f"(下限 {MIN_SUBJECT_AREA_RATIO:.1%}),像散落的噪点而不是角色", + ) + limit = reject_aspect_for(canvas) + if facts.subject_ratio > limit: + raise MasterRejected( + MasterRejectCode.ASPECT_TOO_WIDE, + f"主体 w/h={facts.subject_ratio:.2f} 超过 {limit:.2f};" + "下游画布装不下,再宽只能把角色硬缩成一条,请换一张主体没这么扁的母版", + ) + return facts diff --git a/backend/packages/ai_engine/src/windup_ai_engine/master_prep.py b/backend/packages/ai_engine/src/windup_ai_engine/master_prep.py new file mode 100644 index 00000000..b4e6d3fe --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/master_prep.py @@ -0,0 +1,76 @@ +"""母版规格与预处理:每个动作需要什么样的母版。 + +**核心规律(三次实测验证,写死为契约):母版姿态决定动作,提示词只能微调。** + - walk:母版**朝侧向**才不转身;正面母版配侧走词 → 模型靠转身调和图文矛盾。 + - jump:母版**顶部留白**才不被视频画面裁掉。 + - attack:必须给**极限蓄力母版**(武器已拉到身后腰际)。用站立母版时,即使提示词写死 + "武器不过头顶 / 不转身 / 只做一次",模型仍会抡过头顶、转到背面、劈两次 —— 强动作 + 先验压不住;换蓄力母版后模型只能"接着往前挥",没有再抡起的空间。 + + +实测教训:母版里角色居中、占 ~70% 画面高时,i2v 跳跃会让角色**头顶顶出视频画面上沿** +被裁掉(生成本身没错,是构图没留够空间)。规则同 MasterSpec 的"运动方向多留白": + - jump:向上运动 → 顶部补空间,角色坐低 + - dash / walk / run:向右位移 → 前进方向多留白(由母版生成时构图保证,此处不改) + +纯 PIL,零 API。背景色取母版四角中位色,补出来的边与母版底色一致。 +""" + +from __future__ import annotations + +import io + +from PIL import Image + +from windup_ai_engine._subject import bg_color as _bg_color + +__all__ = ["add_headroom", "prepare_master", "MASTER_POSES"] + +# 各动作所需的母版姿态(生成专用母版时的姿势描述)。空=可直接用中性站立母版。 +MASTER_POSES = { + "walk": "", # 中性站立即可,但必须朝侧向 + "run": "", + "idle": "", + # jump:与 attack 同理——重甲带剑角色的"跳跃"强动作先验压不住(站立母版会让模型摆 + # 造型、只举剑不腾空,实测)。给**极限蓄力半蹲母版**,模型只能"接着往上蹬"。顶部留白 + # 由 prepare_master(add_headroom)保证。 + "jump": ( + "deep crouch coiled to spring straight upward: the knees bent low and the hips sunk down, " + "both arms drawn back behind the body, the weight loaded onto both legs at the very moment " + "before springing straight up, the weapon kept in a fixed grip; " + "leave generous empty space above the head" + ), + "attack": ( + "extreme wind-up stance for a horizontal slash: the weapon drawn far BACK behind the body " + "at WAIST height, the torso twisted back and coiled, weight fully loaded on the back leg, " + "both arms low and pulled back, the weapon staying BELOW the shoulders; " + "leave generous empty space on the swing side" + ), +} + + +def add_headroom(master: bytes, ratio: float = 0.6) -> bytes: + """在母版上方补空间,让角色坐到画面下部,给腾空留出余量。 + + Args: + master: 母版图 bytes。 + ratio: 处理后角色所占的画面高度比例(越小头顶空间越多)。0.6 表示角色高度 + 约占新画面的 60%,上方留约 40%。 + """ + if not 0.1 < ratio < 1.0: + raise ValueError("ratio 需在 (0.1, 1.0) 之间") + img = Image.open(io.BytesIO(master)).convert("RGB") + new_h = max(img.height + 1, int(round(img.height / ratio))) + canvas = Image.new("RGB", (img.width, new_h), _bg_color(img)) + canvas.paste(img, (0, new_h - img.height)) # 原图贴底,空间加在顶部 + buf = io.BytesIO() + canvas.save(buf, "PNG") + return buf.getvalue() + + +def prepare_master(master: bytes, action: str) -> bytes: + """按动作类型预处理母版;不需要处理的动作原样返回。""" + if action in ("jump", "attack"): + # jump 向上腾空、attack 挥砍过头顶,都会顶出视频画面上沿(实测 attack 15/72 帧触顶) + return add_headroom(master, ratio=0.62 if action == "jump" else 0.70) + return master diff --git a/backend/packages/ai_engine/src/windup_ai_engine/ports/__init__.py b/backend/packages/ai_engine/src/windup_ai_engine/ports/__init__.py new file mode 100644 index 00000000..c3eb1684 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/ports/__init__.py @@ -0,0 +1,171 @@ +"""ai_engine 对外契约(ports)—— server 只 import 这里,不碰 slicing / strategy / impl。 + +CI 的 import-linter 分层门禁会强制:app.server 依赖只到 ai_engine.ports。 +换掉内部实现(strategy / provider)时 server 零改动。 + +MVP 边界(与作者对齐):ai_engine **只产出帧 bytes + 进度**,不碰存储 / DB。 +母版(master)由 server 侧从 ``Character.reference_image_url`` 取好、以 bytes 传入; +产出的帧由 server 侧上传对象存储、落 ``character_data``。故本层无 ArtifactStore 依赖。 +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Protocol, runtime_checkable + +from windup_common.models import ActionSpec, CharacterCard + + +# ---- server 实现、注入给 ai_engine 的进度回调 port ---- +class ProgressPort(Protocol): + """进度上报 —— server 转 SSE / 轮询状态(取代管线里的 print)。""" + + def step(self, stage: str, i: int, total: int, note: str = "") -> None: ... + + +# ---- 入口拒绝(在花钱之前)---- +class MasterRejectCode(str, Enum): + """母版被拒的原因 —— server 据此选文案,别用异常消息做分支(消息会改)。 + + 取值全部是**本地零成本可判**的形态问题;判不了的(画的是不是角色、朝向对不对) + 不在此列,见 :mod:`windup_ai_engine.master_check` 的"本层不判什么"。 + """ + + UNDECODABLE = "undecodable" # 不是图 / 截断 / 编码不支持 + NO_SUBJECT = "no_subject" # 全透明或全同色:没有可动的东西 + SUBJECT_TOO_SMALL = "subject_too_small" # 主体小到与噪点/水印无从区分 + ASPECT_TOO_WIDE = "aspect_too_wide" # 主体太扁,方形 cell 里只能压成一条 + + +class MasterRejected(ValueError): + """母版不具备可生成性,在**调用付费模型之前**拒绝。 + + 与 ai_engine 其他异常的分工(这条分工是给 server 用的): + - ``MasterRejected`` = **调用方的输入不行**,同一张母版重试多少次都一样。 + server 应映射成 4xx、把 ``code`` 翻成"请换一张母版"类文案,**不要重试**。 + - ``NotImplementedError`` / 其他 ``ValueError`` = 引擎侧装配或产出出了问题 + (路线没注入、strategy 吐空帧、帧数对不上),属于 5xx、要人介入, + 让用户换母版是把锅甩错地方。 + """ + + def __init__(self, code: MasterRejectCode, detail: str) -> None: + super().__init__(f"母版不可用({code.value}):{detail}") + self.code = code + self.detail = detail + + +# ---- ai_engine 出参(不含存储引用:上传 / 落库在 server 侧)---- +@dataclass(frozen=True) +class ActionQuality: + """这一次出帧的成色 —— 让上层能判"交付 / 重试 / 让用户换母版"。 + + 没有这个,``GeneratedAction`` 只能表达"生成完了",不能表达"生成得怎么样": + 一段**每帧都一样**的 walk 和一段步态干净的 walk,帧数、时长、fps 完全相同, + 调用方分辨不出 —— 本仓吃过四次的正是这类"看起来成功的错结果"。 + + 三个字段各自不可由其他两个推导(下面逐条说明必要性)。刻意**没有**的字段: + - 糊帧率(``slicing.quality.blur_ratio``):2026-08-05 实测 6 段真 i2v + **没有一帧糊帧**,加进来是个恒等于 1 的常数,上层拿它做不了任何决定。 + 真出现糊帧再加,那时才有阈值可依。 + - 抽帧降级原因(``slicing.pick_cycle`` 的三条退化路径):见该函数 docstring 里 + 记的缺口。降级**对交付物的后果**由 ``loop_seam`` 直接测得,而"降级的原因" + 今天没有任何调用方会据此改变行为,故不塞进出参。 + """ + + motion_scale: float + """交付帧的相邻帧平均差异(48×48 灰度绝对尺度)。**0.0 = N 张同一张图。** + + 上层拿它做的决定:接近 0 → 这不是动画,**不要交付**(退款 / 重试 / 提示母版 + 姿态不适合该动作)。它与 ``dead_frames`` 不重复而是互补 —— ``dead_frames`` + 的两条判据都是相对的(比邻居、比自身 p75),整段完全冻结时全部不成立、 + 一帧死帧都报不出(见 ``slicing.quality.motion_scale`` 的实测说明)。 + """ + + dead_frames: tuple[int, ...] + """与前一帧几乎无变化的帧下标(下标 0 不参与判定:它没有前一帧)。 + + 上层拿它做的决定:``len(dead_frames)/len(frames)`` 偏高 → 用户花 N 帧的钱只拿到 + N-K 个不同姿态,提示重试或换母版。给**下标**而不是个数,是因为分布形态对应两种 + 不同的病、修法不同:连续一段 = 动作停住(母版姿态不对 / 视频后半段衰减), + 隔帧散布 = 有效帧率减半(i2v 复制帧),前者换母版、后者调抽帧密度。 + """ + + loop_seam: float | None + """末帧接回首帧的跳幅 ÷ 相邻帧平均步长。1.0 ≈ 接缝与一个正常帧间步长同量级。 + + 上层拿它做的决定:循环类动作(idle/walk/run)会被引擎反复播放,接缝大就是肉眼 + 可见的"跳一下";超过约 1.2 → 提示重试。取归一化值而不是原始差,是为了让不同 + 动作幅度之间可比。 + + ``None`` = **这个数在本次生成里不可读**,两种情形:一次性动作(jump/attack/hit) + 本就不闭环;或 ``motion_scale`` 为 0(整段静止,连"一个正常步长"都没有,归一化 + 无从谈起)。调用方要区分就看 ``motion_scale``,**不要把 None 当 0.0** —— + 0.0 会被读成"完美闭环",正是本仓忌讳的"貌似合理的默认值"。 + """ + + +@dataclass +class GeneratedAction: + """一个动作的生成产物:对齐后的原地序列帧 + 逐帧时长 + 成色。 + + frames / durations **等长**;server 侧把每帧上传对象存储得 URL,组成 + ``CharacterActionOutput.frames[{index, image_url, duration_ms}]`` 回填 character_data。 + """ + + frames: list[bytes] = field(default_factory=list) # RGBA PNG,按播放序 + # 播放时序的**唯一**真相源。曾另有一个 fps 字段抄自入参,与本字段互相矛盾: + # fps=20 宣称 50ms/帧,而 walk 这里给的是 125ms/帧 —— 同一段素材两个播放速度, + # 取哪个看消费方心情(2026-08-10 机器审 P2)。逐帧 ms 严格更能表达(关键帧定格), + # 所以删 fps 保 durations;真要单一帧率,由消费方从本字段算。 + durations: list[int] = field(default_factory=list) # 逐帧时长(ms),与 frames 等长 + # 无默认值、且 kw_only 让它能排在有默认值的字段之后:**不给"没测"留缺省**。 + # 给个 None 缺省的话,漏测与"测出来没问题"在调用方看来一模一样,而这个出参的 + # 全部意义就是把这两者分开。 + quality: ActionQuality = field(kw_only=True) + + +# ---- ai_engine 暴露给 server(server 调用的唯一入口)---- +@runtime_checkable +class CharacterGeneratorPort(Protocol): + """生成入口:角色卡 + 动作规格 + 母版 → 帧序列产物。 + + 不关心租户 / 配额 / 任务状态 / 存储(那些在 app.server)。 + + Args: + card: 角色卡。**当前唯一实现的视频路线一个字段都不读**——``git grep 'card\\.'`` + 在 ai_engine 下零命中(2026-08-08 复核)。这不是遗漏:i2v 的角色身份完全由 + ``master`` 这张母版图像承载,身份描述再写一遍反而会和母版打架。本参数是给 + 未实现路线预留的入参:逐帧图生图(#53)要靠 ``name`` / ``desc`` 在每帧提示词里 + 锁一致性,渲染出帧(#81 #122)要靠 ``master_ref`` / ``version`` 定位 3D 资产。 + **调用方不要指望改 card 能影响视频路线的产出。** + action: 动作规格(类型 / 帧数 / 风格化 / 朝向)。视频路线的实际入参在这里: + ``action``、``n_frames``、``facing``、``stylize`` 等。 + master: 定妆母版图 bytes(server 从 reference_image_url 取)。**视频路线的 + 角色一致性靠它,不靠 card。** 进付费模型之前会先过一遍可生成性预检, + 见 Raises。 + progress: 进度回调。 + canvas: 交付画布 ``(宽, 高)``,单位像素。``None`` = 引擎默认(256 方形)。 + **给上层传项目 sprite 尺寸用的。** 不给的话引擎恒出 256,上层要缩到项目 + 尺寸就得再来一次重采样;而那一步用 ``Image.thumbnail``(只缩不放),放大 + 方向根本不放大、还会把脚线从 0.92 挪到 0.709(2026-08-11 实测),角色不 + 站在地上。让引擎一次出到目标尺寸,那次二次缩放就整个消掉。 + 画布几何按比例定义,故任何尺寸下构图不变、母版预检阈值同样有效。 + + Raises: + MasterRejected: 母版形态不可生成(见 :class:`MasterRejectCode`)。**在花钱 + 之前抛**,同一张母版重试无意义 → server 映射 4xx、请用户换母版。 + NotImplementedError: 该动作分流到的路线没有实现或没注入 strategy。 + ValueError: 产出对不上契约(空帧 / 帧数不足)。钱已经花了,但错产物不放行。 + + 出参的 ``GeneratedAction.quality`` 是**必填**的成色读数:帧数对、无异常并不 + 等于产物可用,调用方交付前应据它决定交付 / 重试 / 让用户换母版。 + """ + + def generate( + self, + card: CharacterCard, + action: ActionSpec, + master: bytes, + progress: ProgressPort, + canvas: tuple[int, int] | None = None, + ) -> GeneratedAction: ... diff --git a/backend/packages/ai_engine/src/windup_ai_engine/postprocess/__init__.py b/backend/packages/ai_engine/src/windup_ai_engine/postprocess/__init__.py new file mode 100644 index 00000000..e69f0ec7 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/postprocess/__init__.py @@ -0,0 +1,28 @@ +"""后处理:把选好的帧落地成交付级序列帧(像素化 / 对齐 / 打包)。 + +抽帧 / 选帧见 :mod:`..slicing`。逐帧时长 ``frame_durations`` 在 :mod:`.rootmotion`。 +""" + +from .rootmotion import DEFAULT_FPS_MS, extract_root_motion, frame_durations +from .pixelate import ( + detect_pixel_size, + extract_palette, + master_pixel_spec, + pixelate_frames, + to_pixel_art, +) +from .pack import align_bottom_center, save_gif, sprite_sheet + +__all__ = [ + "to_pixel_art", + "pixelate_frames", + "detect_pixel_size", + "extract_palette", + "master_pixel_spec", + "extract_root_motion", + "frame_durations", + "DEFAULT_FPS_MS", + "align_bottom_center", + "sprite_sheet", + "save_gif", +] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/postprocess/pack.py b/backend/packages/ai_engine/src/windup_ai_engine/postprocess/pack.py new file mode 100644 index 00000000..2b6207b3 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/postprocess/pack.py @@ -0,0 +1,133 @@ +"""对齐 / 打包（后处理的收尾：脚线对齐 → sprite sheet / gif）。 + +抽帧 / 选帧见 :mod:`..slicing`，像素化见 :mod:`.pixelate`，抠图见 framework 的 +MatteProvider（#20）。本模块把对齐后的帧拼成交付物。 +""" + +from __future__ import annotations + +from PIL import Image + +__all__ = ["CELL", "FILL_H", "FILL_W", "FOOT_LINE", "align_bottom_center", + "sprite_sheet", "save_gif"] + +# 交付画布的几何 —— 提成模块常量而不是只当默认参数,是因为**入口预检要按同一套几何 +# 判母版能不能装下**(见 master_check.REJECT_ASPECT)。抄一份数字过去就等于埋下 +# "改了这里、那边阈值不动"的静默分歧。 +CELL = 256 # 方形 cell 边长(交付序列帧的画布) +FOOT_LINE = 0.92 # 脚线在画布中的高度比例 +FILL_H = 0.62 # 参考姿态占画布高的比例(留余量给举过头顶的动作) +FILL_W = 0.96 # 主体占画布宽的上限(宽度兜底的天花板) + + +def align_bottom_center( + frames: list[Image.Image], + cell: int = CELL, + foot_line: float = FOOT_LINE, + fill_h: float = FILL_H, + fill_w: float = FILL_W, + preserve_lift: bool = False, + ref_height: float | None = None, + cell_h: int | None = None, +) -> list[Image.Image]: + """按脚线对齐到统一画布,消除逐帧画布漂移(Issue #21)。 + + **整段共用一个缩放系数**(取全序列最高帧定标),不逐帧归一化 —— 逐帧各自缩放到等高 + 会把走路自然的身高起伏(实测约 4%)反向变成"忽大忽小":蹲下的帧被放大、伸展的帧被 + 缩小。统一缩放后帧间只剩真实姿态差,尺度稳定。 + + 水平方向按**主体水平中心**对齐(不含挥出的武器会更好,当前用整体包围盒中心兜底); + 垂直方向按**脚线**(包围盒底边)对齐到 ``foot_line``。 + + ``ref_height``:**跨动作一致性的关键**,单位=传入帧的像素高。给定时按它定标,否则按本 + 序列最高帧。按最高帧定标会让"举过头顶"的动作整段被缩小去迁就那一帧 —— 实测攻击时 + 斧头高举使 bbox 从 485 涨到 660,角色本体因此明显变小;跳跃顶点同理。故传入**参考姿态** + (站立)的高度,各动作即共用同一本体尺寸。``fill_h`` 默认 0.62,给举过头顶留出余量。 + + ``preserve_lift``:腾空位移**默认不烘进像素**(业界:位移交引擎 root motion)。仅在要把 + 位移画进序列帧时才开;开启后以序列里最低的脚线为地面基准,保留每帧相对地面的抬升量。 + + ``cell``/``cell_h``:交付画布的宽与高,``cell_h=None`` 即方形 ``cell×cell``(默认, + 行为与加这个参数之前逐像素相同)。**要能出非方形画布,是为了让引擎一次就出到项目 + 要的 sprite 尺寸、不必在上层再缩一次。** 上层那次二次缩放不是"糊一点"那么简单: + 它用 ``Image.thumbnail`` 补边,而 thumbnail **只缩不放** —— 项目要 512 时 256 的帧 + 根本不会被放大,而是原尺寸居中贴进 512 画布,于是这里刚对齐好的脚线(0.92)被挪到 + 0.709(2026-08-11 实测),角色不站在地上了,跨动作对齐也一起失效。 + + 几何按"比例"而不是"像素"表达(``foot_line``/``fill_h``/``fill_w`` 都是比例),所以 + 换画布尺寸不改变构图,母版入口预检(``master_check.REJECT_ASPECT`` = 2*FILL_W/FILL_H) + 与出帧仍共用同一套几何 —— 那条阈值里没有 cell,本来就与画布像素尺寸无关。 + """ + import numpy as np + + cw = cell + ch = cell if cell_h is None else cell_h + if cw < 1 or ch < 1: + # 不静默出一张 0×0:PIL 允许建 0 边长的图,后面 alpha_composite 也不报错, + # 错产物要到落库/前端才暴露。 + raise ValueError(f"交付画布尺寸必须为正,收到 cell={cell} cell_h={cell_h}") + + boxes: list[tuple[int, int, int, int] | None] = [] + for f in frames: + ys, xs = np.where(np.asarray(f)[:, :, 3] > 128) + boxes.append( + (int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1) + if len(ys) + else None + ) + heights = [b[3] - b[1] for b in boxes if b] + if not heights: + return [Image.new("RGBA", (cw, ch), (0, 0, 0, 0)) for _ in frames] + # 腾空模式:以最低脚线(数值最大 = 站在地上)为地面基准,保留每帧的抬升量 + ground = max(b[3] for b in boxes if b) if preserve_lift else 0 + # 定标要把抬升量算进去,否则跳到最高时头顶会顶出画布被切掉 + if preserve_lift: + need = max((ground - b[3]) + (b[3] - b[1]) for b in boxes if b) + scale = (ch * fill_h) / max(1, need) + elif ref_height: + scale = (ch * fill_h) / ref_height # 参考姿态定标(跨动作一致) + else: + scale = (ch * fill_h) / max(heights) # 回退:本序列最高帧 + + # 宽度兜底:上面三条分支**只按高度定标** —— 这是"主体是纵向长条"的人形先验。 + # 横向长条主体(四足兽/坐骑/龙)按同一系数缩放后宽度超过画布宽,会被下面的 + # alpha_composite 以负 dest **静默切掉**左右(PIL 不报错,直接丢像素)。 + # 裁切悬崖 = 主体 w/h > 1/fill_h ≈ 1.61。实测(2026-08-05):狐狸母版 w/h=1.78 + # 丢 27px(鼻尖+尾尖);w/h=2.0 只剩 79.9% 内容;狼/马常见 2.0-2.5 → 丢 19%-35% 体宽。 + # 人形 w/h≈0.3-1.1 时该约束**恒不生效**,故人形产物逐像素不变。 + widths = [b[2] - b[0] for b in boxes if b] + scale = min(scale, (cw * fill_w) / max(1, max(widths))) + + out = [] + for f, box in zip(frames, boxes): + if box is None: + out.append(Image.new("RGBA", (cw, ch), (0, 0, 0, 0))) + continue + crop = f.crop(box) + w = max(1, round(crop.width * scale)) + h = max(1, round(crop.height * scale)) + crop = crop.resize((w, h), Image.NEAREST) + lift = round((ground - box[3]) * scale) if preserve_lift else 0 + canvas = Image.new("RGBA", (cw, ch), (0, 0, 0, 0)) + canvas.alpha_composite(crop, (cw // 2 - w // 2, int(ch * foot_line) - h - lift)) + out.append(canvas) + return out + + +def sprite_sheet(frames: list[Image.Image], bg=(0, 0, 0, 0)) -> Image.Image: + """横向拼接为 sprite sheet。""" + if not frames: + raise ValueError("frames 为空") + w, h = frames[0].size + sheet = Image.new("RGBA", (w * len(frames), h), bg) + for i, f in enumerate(frames): + sheet.alpha_composite(f.convert("RGBA"), (i * w, 0)) + return sheet + + +def save_gif(frames: list[Image.Image], path: str, duration: int = 120) -> None: + """导出循环 gif 供预览。""" + if not frames: + raise ValueError("frames 为空") + rgba = [f.convert("RGBA") for f in frames] + rgba[0].save(path, save_all=True, append_images=rgba[1:], duration=duration, loop=0, disposal=2) diff --git a/backend/packages/ai_engine/src/windup_ai_engine/postprocess/pixelate.py b/backend/packages/ai_engine/src/windup_ai_engine/postprocess/pixelate.py new file mode 100644 index 00000000..4910fa96 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/postprocess/pixelate.py @@ -0,0 +1,252 @@ +"""像素化后处理:把生成帧转成脆边限色的像素精灵。 + +视频路线实测(Issue #35): +- i2v 能解决步态(腿真交替、不转身);对**插画风**角色它保留插画质感 → 需要像素化转风格。 +- 对**原生像素**角色 i2v 其实能保住像素感,但链路上两道有损压缩(首帧 JPG q90 + 视频 H.264) + 会在硬边处产生振铃噪点(表现为灰颗粒),像素越细越明显;而通用的"降采样 + 32 色量化" + 因为**网格对不齐**反而更糊。 +- 解法:有母版时按 :func:`master_pixel_spec` 量出母版的**原生像素块大小**与**真实色板**, + 按母版网格降采样 + 颜色吸附回母版色板 —— 压缩灰颗粒不属于色板,会被强制消掉。 + +纯 Pillow / numpy,零 API、秒级,符合"本机只做轻量 CV"的算力约束。 +输入约定:RGBA 图(alpha 为主体掩码,抠图见 framework 的 MatteProvider / Issue #20)。 +""" + +from __future__ import annotations + +import numpy as np +from PIL import Image + +__all__ = [ + "to_pixel_art", + "pixelate_frames", + "detect_pixel_size", + "extract_palette", + "master_pixel_spec", +] + + +def _content_bbox(rgba: Image.Image, alpha_thr: int = 128) -> tuple[int, int, int, int]: + """求主体包围盒。 + + 用 :func:`_subject_mask` 而非只看 alpha:母版常是**不透明白底**,只看 alpha 会把整张 + 画布当主体,导致逻辑像素高被算成整图高而非角色高(实测踩过)。 + """ + mask = _subject_mask(rgba.convert("RGBA"), alpha_thr) + ys, xs = np.where(mask) + if len(ys): + return int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1 + return 0, 0, rgba.width, rgba.height + + +def _axis_block_size(crop: np.ndarray, axis: int, min_delta: int, min_frac: float) -> int: + """沿 ``axis`` 估块边长:显著色变位置 → 合并相邻 → 取最常见间距。""" + d = np.abs(np.diff(crop, axis=axis)).sum(axis=2) + frac = (d > min_delta).mean(axis=1 - axis) + edges = np.flatnonzero(frac > min_frac) + 1 + if len(edges) < 3: + return 1 + # 块边界常因轻微抗锯齿占相邻两行/列,合并成一条,否则 gap=1 会淹没真实值 + edges = edges[np.concatenate([[True], np.diff(edges) > 1])] + gaps = np.diff(edges) + gaps = gaps[gaps >= 2] + return int(np.bincount(gaps).argmax()) if len(gaps) else 1 + + +def detect_pixel_size( + img: Image.Image, min_delta: int = 30, min_frac: float = 0.02, max_size: int = 64 +) -> int: + """检测像素画的原生像素块边长(非像素画/检测不出时返回 1)。 + + 原理:像素画的色块边界落在同一网格上,相邻边界间距 = 块边长的整数倍,故取 + **最常见间距**即块边长。两轴分别估,取较小者(更保守,宁可细不可糊)。 + """ + rgba = img.convert("RGBA") + x0, y0, x1, y1 = _content_bbox(rgba) + crop = np.asarray(rgba.crop((x0, y0, x1, y1)).convert("RGB")).astype(np.int16) + if crop.size == 0: + return 1 + sizes = [_axis_block_size(crop, ax, min_delta, min_frac) for ax in (0, 1)] + best = min(s for s in sizes) if all(s >= 1 for s in sizes) else 1 + return max(1, min(best, max_size)) + + +def _erode(mask: np.ndarray, k: int) -> np.ndarray: + """二值腐蚀 k 次(纯 numpy 移位,不引 scipy)。""" + m = mask + for _ in range(max(0, k)): + m = ( + m + & np.roll(m, 1, 0) + & np.roll(m, -1, 0) + & np.roll(m, 1, 1) + & np.roll(m, -1, 1) + ) + if not m.any(): + return mask + return m + + +def _subject_mask( + rgba: Image.Image, alpha_thr: int = 128, bg_tol: int = 40, erode: int = 0 +) -> np.ndarray: + """主体掩码:优先用真实 alpha;母版常是**不透明白底**,此时按四角背景色排除背景。 + + 两个实测踩过的坑: + 1. 不排背景 → 白底占多数像素、吃光色板名额 → 角色被整体吸附成白色。 + 2. 排了背景但保留边缘 → 角色/白底之间的**抗锯齿过渡色**(近白)混进色板 → + 视频里的浅色噪点就近吸附成白点,满身白斑。故取色板时用 ``erode`` 腐蚀掉边缘。 + """ + arr = np.asarray(rgba) + alpha = arr[:, :, 3] + if not alpha.min() > alpha_thr: # 有真实抠图 + mask = alpha > alpha_thr + else: + rgb = arr[:, :, :3].astype(np.int16) + corners = np.stack([rgb[0, 0], rgb[0, -1], rgb[-1, 0], rgb[-1, -1]]) + bg = np.median(corners, axis=0) + mask = np.abs(rgb - bg).sum(axis=2) > bg_tol + return _erode(mask, erode) + + +def extract_palette( + img: Image.Image, max_colors: int = 32, alpha_thr: int = 128, erode: int = 3 +) -> np.ndarray: + """提取母版真实色板,返回 (K,3) uint8。 + + 只统计主体像素(见 :func:`_subject_mask`,并腐蚀掉抗锯齿边缘),再用中位切分量化 + 归并噪声色 —— 生成的"像素画"常带轻微噪点/抗锯齿,同一名义色被打散成大量近似色, + 直接按频率统计会全被当杂色滤掉。 + """ + rgba = img.convert("RGBA") + arr = np.asarray(rgba) + mask = _subject_mask(rgba, alpha_thr, erode=erode) + pixels = arr[:, :, :3][mask] + if not len(pixels): + pixels = arr[:, :, :3].reshape(-1, 3) + strip = Image.fromarray(pixels.reshape(1, -1, 3).astype(np.uint8), "RGB") + quant = strip.quantize(colors=max(2, max_colors), method=Image.MEDIANCUT) + pal = np.asarray(quant.getpalette()[: max(2, max_colors) * 3], dtype=np.uint8).reshape(-1, 3) + used = np.unique(np.asarray(quant)) + return pal[used[used < len(pal)]] + + +def master_pixel_spec(master: Image.Image, max_colors: int = 48) -> tuple[int, np.ndarray]: + """从母版量出 (角色的逻辑像素高, 母版色板)。 + + 逻辑像素高 = 母版里角色占的像素行数 ÷ 原生像素块边长 —— 即"这个角色本来是多少 + 像素高的精灵"。用它当 ``target_h`` 可自动吸附网格,不必人肉猜分辨率。 + + ``max_colors`` 实测取值:32 太少 —— 中位切分按面积分箱,大面积色(如裸腿肤色/棕靴) + 会挤占名额,小面积但需渐变的衣服色档位不足 → 中间调就近吸到邻近色相(绿衣泛橄榄黄); + 96 太多 —— 抗锯齿近白色重新拿到独立分箱 → 边缘冒白噪点。48 是实测的安全区。 + """ + x0, y0, x1, y1 = _content_bbox(master.convert("RGBA")) + block = detect_pixel_size(master) + logical_h = max(1, round((y1 - y0) / block)) + return logical_h, extract_palette(master, max_colors=max_colors) + + +def _to_perceptual(rgb: np.ndarray) -> np.ndarray: + """RGB → 近似感知空间(亮度 + 两个色差轴),float32。 + + 直接在 RGB 里取最近邻会**跳色相**:绿衣的中间调可能被吸到橄榄黄(实测踩过)。 + 换成亮度/色差轴并给色差加权后,同色相内的明暗过渡优先匹配,色相跳变被压住。 + 这里用 YCbCr 型线性变换(比 Lab 便宜得多,足够拉开色相)。 + """ + f = rgb.astype(np.float32) + r, g, b = f[..., 0], f[..., 1], f[..., 2] + y = 0.299 * r + 0.587 * g + 0.114 * b + cb = b - y + cr = r - y + w = 2.0 # 色差权重 >1:宁可亮度差一点,也别换色相 + return np.stack([y, w * cb, w * cr], axis=-1) + + +def _snap_to_palette(rgb: np.ndarray, palette: np.ndarray) -> np.ndarray: + """把每个像素吸附到色板中最近的颜色(感知空间最近邻,分块避免大内存)。 + + 用 float32 感知空间:①避免 int16 平方距离溢出(255² > 32767,实测让绿衣变肉色); + ②按色相优先匹配,防止 RGB 空间里的跨色相跳变。 + """ + flat = _to_perceptual(rgb).reshape(-1, 3) + pal_p = _to_perceptual(palette).reshape(-1, 3) + pal_rgb = palette.astype(np.uint8).reshape(-1, 3) + out = np.empty((len(flat), 3), dtype=np.uint8) + step = 65536 + for i in range(0, len(flat), step): + chunk = flat[i : i + step] + d = ((chunk[:, None, :] - pal_p[None, :, :]) ** 2).sum(axis=2) + out[i : i + step] = pal_rgb[d.argmin(axis=1)] + return out.reshape(rgb.shape) + + +def to_pixel_art( + rgba: Image.Image, + target_h: int = 100, + palette_size: int = 32, + alpha_thr: int = 128, + palette: np.ndarray | None = None, +) -> Image.Image: + """单帧转像素风,返回小尺寸 RGBA(``target_h`` 高,等比宽)。 + + 步骤:裁到主体包围盒 → 等比缩到 ``target_h``(NEAREST 网格降采样)→ 限色。 + 限色两种模式: + - ``palette`` 给定(推荐,原生像素角色):**吸附到母版真实色板**,顺带消掉 + JPG/H.264 在硬边留下的灰颗粒。 + - ``palette=None``(插画转像素):按 ``palette_size`` 做八叉树量化。 + + Args: + target_h: 目标像素高;原生像素角色建议用 :func:`master_pixel_spec` 算出的逻辑高。 + palette_size: 无母版色板时的量化色数。 + palette: (K,3) uint8 母版色板。 + """ + if target_h < 1: + raise ValueError("target_h 必须 >= 1") + rgba = rgba.convert("RGBA") + x0, y0, x1, y1 = _content_bbox(rgba, alpha_thr) + crop = rgba.crop((x0, y0, x1, y1)) + w, h = crop.size + target_w = max(1, round(w * target_h / h)) + small = crop.resize((target_w, target_h), Image.NEAREST) + + alpha = np.asarray(small)[:, :, 3] + if palette is not None and len(palette): + rgb = _snap_to_palette(np.asarray(small.convert("RGB")), palette) + else: + rgb = np.asarray( + small.convert("RGB") + .quantize(colors=max(2, palette_size), method=Image.FASTOCTREE) + .convert("RGB") + ) + out = np.dstack([rgb, alpha]).astype(np.uint8) + return Image.fromarray(out, "RGBA") + + +def pixelate_frames( + frames: list[Image.Image], + target_h: int = 100, + palette_size: int = 32, + palette: np.ndarray | None = None, + ref_height: float | None = None, +) -> list[Image.Image]: + """批量像素化一组帧,**整段共用一个缩放系数**,便于打包为 sprite sheet。 + + ``target_h`` 是**基准姿态**的目标像素高,其余帧按同一系数等比缩放 —— 不是把每帧都拉 + 到等高。逐帧拉等高会把走路自然的身高起伏反向变成"忽大忽小"(实测踩过:蹲下的帧被放大)。 + + ``ref_height``:**跨动作一致性的关键**。给定时用它当基准(单位=源图像素),否则用本序列 + 最高帧。同一角色的各个动作若各自取自己的最高帧定标,切换状态时角色会忽大忽小 —— + 传入同一个基准(如母版姿态的角色高)即可让 idle/walk/jump/attack 共用一套尺度。 + """ + if not frames: + return [] + box_h = [] + for f in frames: + _, y0, _, y1 = _content_bbox(f.convert("RGBA")) + box_h.append(max(1, y1 - y0)) + scale = target_h / (ref_height if ref_height else max(box_h)) + return [ + to_pixel_art(f, max(1, round(h * scale)), palette_size, palette=palette) + for f, h in zip(frames, box_h) + ] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/postprocess/rootmotion.py b/backend/packages/ai_engine/src/windup_ai_engine/postprocess/rootmotion.py new file mode 100644 index 00000000..88487645 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/postprocess/rootmotion.py @@ -0,0 +1,69 @@ +"""Root motion(位移轨迹)与逐帧时长 —— 按 2D 游戏业界惯例分离"姿势"与"位移"。 + +业界做法(调研 2026-07-28): +- **位移不烘进序列帧**。连续位移动作几乎一律用 *in-place animation + 引擎代码驱动移动*, + 因为玩家要即时操控:跑动中转向应立刻响应,而不是等一段烘死的位移播完。平台游戏的跳跃 + 也是"几个姿势定格 + 引擎物理驱动上下",不是把抛物线画进像素。 + → 序列帧保持**原地**(脚线对齐),位移单独作为 root-motion 轨道交给引擎。 +- **逐帧时长比帧数更重要**("frame timing beats frame count")。业界常用: + idle 400–500ms/帧、walk 100–150ms、run 80–100ms、attack 起手 80–100ms 且**触点定格 + 150–200ms**。全程等时长会让动作发飘、没有重量感。 + +本模块只做几何与时长计算,纯 numpy,零 API。 +""" + +from __future__ import annotations + +import numpy as np +from PIL import Image + +__all__ = ["extract_root_motion", "frame_durations", "DEFAULT_FPS_MS"] + +# 各动作的基准单帧时长(ms),取业界常用区间的中值。 +DEFAULT_FPS_MS = { + "idle": 450, + "walk": 125, + "run": 90, + "jump": 110, + "attack": 90, + "hit": 90, +} + + +def extract_root_motion(frames: list[Image.Image], alpha_thr: int = 128) -> list[tuple[int, int]]: + """逐帧相对首帧的 (dx, dy) 位移,单位=像素,y 向上为正。 + + 以主体包围盒的**底边中心**(脚点)为参考点。序列帧本身保持原地时,这条轨道就是引擎 + 要施加的 root motion:jump 的 dy 是腾空高度,walk 的 dx 是前进量。 + """ + pts: list[tuple[float, float]] = [] + for f in frames: + a = np.asarray(f.convert("RGBA")) + ys, xs = np.where(a[:, :, 3] > alpha_thr) + pts.append(((xs.min() + xs.max()) / 2, float(ys.max())) if len(ys) else (np.nan, np.nan)) + arr = np.array(pts, dtype=np.float32) + if np.isnan(arr).any(): # 空帧用邻近值补 + idx = np.arange(len(arr)) + for c in range(2): + good = ~np.isnan(arr[:, c]) + arr[:, c] = np.interp(idx, idx[good], arr[good, c]) if good.any() else 0.0 + base = arr[0] + return [(int(round(p[0] - base[0])), int(round(base[1] - p[1]))) for p in arr] + + +def frame_durations( + action: str, n_frames: int, key_frame: int | None = None, hold_ms: int = 180 +) -> list[int]: + """逐帧时长(ms)。关键帧(触点 / 顶点)加长定格,其余用该动作的基准时长。 + + Args: + action: 动作名(取 :data:`DEFAULT_FPS_MS` 的基准时长,未知动作按 walk)。 + n_frames: 帧数。 + key_frame: 要定格的帧下标(attack 的触点、jump 的顶点);None 表示全程等时长。 + hold_ms: 关键帧时长,业界常用 150–200ms。 + """ + base = DEFAULT_FPS_MS.get(action, DEFAULT_FPS_MS["walk"]) + out = [base] * max(0, n_frames) + if key_frame is not None and 0 <= key_frame < n_frames: + out[key_frame] = max(base, hold_ms) + return out diff --git a/backend/packages/ai_engine/src/windup_ai_engine/prompt/__init__.py b/backend/packages/ai_engine/src/windup_ai_engine/prompt/__init__.py new file mode 100644 index 00000000..dba82745 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/prompt/__init__.py @@ -0,0 +1,15 @@ +"""prompt:各动作的生成提示词与装配。""" + +from .actions import build_attack_prompt, build_idle_prompt +from .jump import JUMP_PHASES, build_jump_prompt +from .walk import WALK_BODY_FRONT, WALK_BODY_SIDE, build_walk_prompt + +__all__ = [ + "WALK_BODY_SIDE", + "WALK_BODY_FRONT", + "build_walk_prompt", + "JUMP_PHASES", + "build_jump_prompt", + "build_idle_prompt", + "build_attack_prompt", +] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/prompt/actions.py b/backend/packages/ai_engine/src/windup_ai_engine/prompt/actions.py new file mode 100644 index 00000000..07116470 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/prompt/actions.py @@ -0,0 +1,87 @@ +"""待机 / 攻击 i2v 提示词。 + +措辞迁自 windup-pipeline 已验证的 prompt_library(idle / slash),按本模块的 facing 分流改写。 + +- **idle**:循环类(tail_match)。只写躯干呼吸节律,武器与双脚显式锁定 —— 逐帧生成待机 + 只会抖不会呼吸,故走 i2v 或程序化 Idle-B。 +- **attack**:一次性类。四条已验证的锁定:①"one single committed motion"防复读; + ②剑长与握点固定;③剑在身前、刃面朝观者(防 Z 轴穿模与刀刃翻转);④终态回戒备并保持。 + 节奏(蓄力慢/挥砍快/触点定格)在抽帧做,不写进 prompt。 +""" + +from __future__ import annotations + +from windup_common.models import Facing + +__all__ = ["build_idle_prompt", "build_attack_prompt"] + +_IDLE_SIDE = ( + "The character stands in place, seen from the side facing right: the chest breathes in one " + "slow, even rhythm, the ribcage expanding and easing back while the shoulders stay level and " + "settled at the same height, the torso rising and lowering in that same slow rhythm, " + "{weapon} resting steady at the side in a fixed grip, {garment} hanging and swaying in the " + "same rhythm, both boots planted firmly on the ground, weight centered, the character stays " + "in the same spot and keeps facing right." +) + +_IDLE_FRONT = ( + "The character stands in place facing the viewer: the chest breathes in one slow, even " + "rhythm, the ribcage expanding and easing back while the shoulders stay level and settled at " + "the same height, the torso rising and lowering in that same slow rhythm, {weapon} resting " + "steady at the side in a fixed grip, {garment} hanging and swaying in the same rhythm, both " + "boots planted firmly on the ground, weight centered, the character keeps FACING THE VIEWER " + "and stays in the same spot." +) + +_ATTACK_SIDE = ( + "Seen from the side facing right, the character makes ONE single committed attack, staying in " + "STRICT SIDE VIEW the whole time: starting coiled with the weight on the back foot, the body " + "leans forward and the weight surges onto the front foot, the arm sweeping {weapon} through " + "one smooth downward crescent arc from high behind the shoulder down across the front to full " + "extension low, {weapon} keeping its exact length and grip position and staying clearly in " + "front of the body with its flat side facing the viewer the whole way, {garment} swinging with " + "the motion, then the body settles back upright into guard and holds that stance, standing " + "steady. The torso and hips keep pointing to the right the entire time and the character never " + "turns toward or away from the viewer." +) + +_ATTACK_FRONT = ( + "Facing the viewer, the character makes ONE single committed attack: starting coiled with the " + "weight on the back foot, the whole body uncoils forward, the arm sweeping {weapon} through " + "one smooth arc across the front to full extension, {weapon} keeping its exact length and grip " + "position and staying clearly in front of the body with its flat side facing the viewer the " + "whole way, {garment} swinging with the motion, then the body settles back upright into guard " + "and holds that stance, standing steady and keeping FACING THE VIEWER." +) + +DEFAULT_WEAPON = "the sword" +DEFAULT_GARMENT = "the cape" + + +def _build( + side: str, front: str, weapon: str, garment: str, feet: str, facing: Facing | str +) -> str: + # 非法值在此炸掉,别静默落到 FRONT 模板(理由见 walk.py 同处注释)。 + tpl = side if Facing(facing) is Facing.SIDE else front + body = tpl.format(weapon=weapon, garment=garment) + return body.replace("boot", feet) if feet != "boot" else body + + +def build_idle_prompt( + weapon: str = DEFAULT_WEAPON, + garment: str = DEFAULT_GARMENT, + feet: str = "boot", + facing: Facing | str = Facing.SIDE, +) -> str: + """待机正文(循环类)。``facing`` 须与母版朝向一致。""" + return _build(_IDLE_SIDE, _IDLE_FRONT, weapon, garment, feet, facing) + + +def build_attack_prompt( + weapon: str = DEFAULT_WEAPON, + garment: str = DEFAULT_GARMENT, + feet: str = "boot", + facing: Facing | str = Facing.SIDE, +) -> str: + """攻击正文(一次性类)。``facing`` 须与母版朝向一致。""" + return _build(_ATTACK_SIDE, _ATTACK_FRONT, weapon, garment, feet, facing) diff --git a/backend/packages/ai_engine/src/windup_ai_engine/prompt/jump.py b/backend/packages/ai_engine/src/windup_ai_engine/prompt/jump.py new file mode 100644 index 00000000..b60cdd2a --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/prompt/jump.py @@ -0,0 +1,64 @@ +"""跳跃 i2v 提示词(一次性动作,非循环)。 + +与 walk/run 的根本差别: +- **不循环**。跳跃是一段有始有终的动作,不能像步态那样抽单周期闭环。 +- **要拆状态**。游戏里跳跃是状态机:蓄力 → 上升 → 顶点 → 下降 → 落地缓冲;悬空时长由 + 物理决定、上升中可被打断,所以必须能分段播放,不能烘成一整段。 +- 提示词要写**"只做一次 + 终态保持"**,防 5s 内复读跳第二次(实测:写了仍会复读,故抽帧层 + 另有 first_action_end 兜底)。 +- **原地起跳、幅度适中**:水平位移交引擎做 root-motion,不烘进像素;幅度过大会让角色顶出 + 视频画面,且序列帧里角色被缩得很小。 + +朝向同 walk:必须与母版一致(side 横版 / front 俯视·2.5D)。 +""" + +from __future__ import annotations + +from windup_common.models import Facing + +__all__ = ["JUMP_BODY_SIDE", "JUMP_BODY_FRONT", "JUMP_PHASES", "build_jump_prompt"] + +# 跳跃的五个状态(引擎侧按这个切段;顺序即时间顺序)。 +JUMP_PHASES = ("crouch", "rise", "apex", "fall", "land") + +JUMP_BODY_SIDE = ( + "The character performs ONE single jump in place, seen from the side facing right: " + "first the knees bend deep into a crouch and the arms drop back, then both boots push " + "off the ground and the whole body lifts straight upward a modest height with the legs " + "tucking up, the body reaches the top of the jump and hangs there for an instant with {garment} " + "floating upward, then the body falls back down with the legs reaching for the ground, " + "and both boots land together with the knees bending to absorb the impact, the weapon " + "stays held steady in a fixed grip the whole time. The character does this ONCE and " + "then stays standing upright in the landing spot, staying centered in frame." +) + +JUMP_BODY_FRONT = ( + "The character performs ONE single jump in place, facing the viewer: first the knees " + "bend deep into a crouch and the arms drop back, then both boots push off the ground " + "hard and the whole body launches straight upward with the knees tucking up toward the " + "camera, the body reaches the top of the jump and hangs there for an instant with " + "{garment} floating upward, then the body falls back down with the legs reaching for " + "the ground, and both boots land together with the knees bending to absorb the impact, " + "the weapon stays held steady in a fixed grip the whole time. The character keeps " + "FACING THE VIEWER, does this ONCE and then stays standing upright, centered in frame." +) + +DEFAULT_GARMENT = "the cape and tabard" + + +def build_jump_prompt( + garment: str = DEFAULT_GARMENT, feet: str = "boot", facing: Facing | str = Facing.SIDE +) -> str: + """按角色装备 + 母版朝向生成跳跃正文。 + + Args: + garment: 起跳时上飘的衣饰。 + feet: 落脚部件用词(替换 boot)。 + facing: :class:`Facing` 成员(或其等价字符串),**必须与母版朝向一致**。 + """ + facing = Facing(facing) # 非法值在此炸掉,别静默落到 FRONT 模板(理由见 walk.py 同处注释) + template = JUMP_BODY_SIDE if facing is Facing.SIDE else JUMP_BODY_FRONT + body = template.format(garment=garment) + if feet != "boot": + body = body.replace("boot", feet) + return body diff --git a/backend/packages/ai_engine/src/windup_ai_engine/prompt/walk.py b/backend/packages/ai_engine/src/windup_ai_engine/prompt/walk.py new file mode 100644 index 00000000..3597e77c --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/prompt/walk.py @@ -0,0 +1,61 @@ +"""走路 i2v 提示词(视频路线)。 + +实测要点(Issue #35): +- 只写正向词、逐条写腿部可见动作(抬 / 摆 / 蹬 / 承重),锁死手持武器不乱动。 +- **提示词的朝向必须与母版朝向一致**。给正面母版喂侧走词(STRICT SIDE)会让模型靠"转身" + 调和图文矛盾——早期"正面母版必转身"的结论正是这么造成的。故按 facing 分流: + side(横版侧走)/ front(俯视·2.5D 朝观者行进),对应 Project.perspective。 +- "半侧"母版(头侧脸 + 身体略正)配 side 词,实测会被自然解析成正侧面走,不转身,够用。 +- 换角色只替换装备子句(如 骷髅:boot→骨足、cape→围巾),机制词保持不变。 +""" + +from __future__ import annotations + +from windup_common.models import Facing + +__all__ = ["WALK_BODY_SIDE", "WALK_BODY_FRONT", "DEFAULT_GARMENT", "build_walk_prompt"] + +# 侧走(横版):整体向右推进 + 锁侧视。 +WALK_BODY_SIDE = ( + "The character walks steadily to the right through the open space, the whole body " + "advancing with every stride: the front boot lifts, swings forward and plants heel " + "first, the rear boot pushes off the ground, the hips and torso carry the weight " + "forward over the planted foot, {garment} swing with the steps, the weapon stays held " + "low and steady at the side in a fixed grip, the upper body stays calm and upright, " + "SIDE VIEW facing right the whole time, the legs clearly visible." +) + +# 正面走(俯视 / 2.5D):朝观者原地行进,身体始终正对观者、不转身。 +WALK_BODY_FRONT = ( + "The character walks in place toward the viewer, marching forward on the spot: each " + "boot lifts, swings forward and plants down in turn while the other pushes off, the " + "knees rise alternately toward the camera, the hips and shoulders sway naturally with " + "each step, {garment} sway with the steps, the weapon stays held low and steady in a " + "fixed grip, the upper body stays calm and upright, the character keeps FACING THE " + "VIEWER the whole time and stays centered in frame, both legs clearly visible." +) + +# 每个角色只替换 garment / feet 两处装备子句,机制词不动。 +DEFAULT_GARMENT = "the cape and tabard" + + +def build_walk_prompt( + garment: str = DEFAULT_GARMENT, feet: str = "boot", facing: Facing | str = Facing.SIDE +) -> str: + """按角色装备 + 母版朝向生成走路正文。 + + Args: + garment: 随步伐摆动的衣饰(如 "the cape and tabard" / "the red scarf and tabard")。 + feet: 落脚部件用词(如 "boot" / "bare bony foot"),替换机制句里的 boot。 + facing: :class:`Facing` 成员(或其等价字符串)。**必须与母版朝向一致**, + 否则模型会靠转身调和矛盾。 + """ + # 注解不是运行期约束:build_* 是普通函数,传 "sidee" 仍进得来。这里显式过一遍 + # Facing() 构造,非法值抛 ValueError —— 若改成 `if facing == Facing.SIDE else FRONT` + # 的二分,"sidee" 会静默落到 FRONT 模板,拿到一段正面走的视频却没有任何报错。 + facing = Facing(facing) + template = WALK_BODY_SIDE if facing is Facing.SIDE else WALK_BODY_FRONT + body = template.format(garment=garment) + if feet != "boot": + body = body.replace("boot", feet) + return body diff --git a/backend/packages/ai_engine/src/windup_ai_engine/slicing/__init__.py b/backend/packages/ai_engine/src/windup_ai_engine/slicing/__init__.py new file mode 100644 index 00000000..d00d4ed6 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/slicing/__init__.py @@ -0,0 +1,36 @@ +"""slicing:视频 → 帧序列。抽帧(extract)+ 选帧(周期 loop / 一次性 oneshot)。 + +视频路线里"从连续视频里挑出交付用的那几帧"这一步:循环类动作抽单步态周期(无缝 +loop),一次性动作裁动作区间。像素化 / 对齐 / 打包在 :mod:`..postprocess`。 + +:mod:`.quality` 原本纯做诊断,现在还兼一份出参职责:交付帧的成色读数 +(``motion_scale`` / ``dead_frame_indices`` / ``loop_seam``)汇成 ``ports.ActionQuality``。 +注意它**仍然不参与选帧** —— 那条消融结论没变,见 :func:`.loop.pick_cycle`。 +""" + +from .extract import extract_all_frames_bytes, extract_frames_bytes +from .loop import find_period, pick_cycle +from .oneshot import ( + find_motion_span, + first_action_end, + foot_line_series, + pick_oneshot, + split_jump_phases, +) +from .quality import dead_frame_indices, loop_seam, motion_scale + +__all__ = [ + "extract_frames_bytes", + "extract_all_frames_bytes", + "find_period", + "pick_cycle", + # 交付成色的三个读数(汇成 ports.ActionQuality;其余 quality.* 仍是内部诊断) + "dead_frame_indices", + "loop_seam", + "motion_scale", + "find_motion_span", + "first_action_end", + "foot_line_series", + "pick_oneshot", + "split_jump_phases", +] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/slicing/_frames.py b/backend/packages/ai_engine/src/windup_ai_engine/slicing/_frames.py new file mode 100644 index 00000000..81066126 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/slicing/_frames.py @@ -0,0 +1,20 @@ +"""选帧与帧质量共用的取样原语。 + +``loop``(选帧)与 ``quality``(诊断)必须在**同一尺度**上看帧,否则两边算出的差异量 +不可比 —— 之前两处各自持有一份 ``_gray`` 与 ``_SMALL``,调一边不会波及另一边, +是一个只会在数据上体现、不会报错的隐患。此处收成唯一定义。 +""" +from __future__ import annotations + +import numpy as np +from PIL import Image + +__all__ = ["SMALL", "gray"] + +# 帧比对统一降采样到 48×48 灰度:够分辨姿态差异,又让全帧对距离矩阵的开销可接受。 +SMALL = 48 + + +def gray(frames: list[Image.Image]) -> list[np.ndarray]: + """帧序列 → 定尺灰度矩阵列表(float32)。""" + return [np.asarray(f.convert("L").resize((SMALL, SMALL)), dtype=np.float32) for f in frames] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/slicing/extract.py b/backend/packages/ai_engine/src/windup_ai_engine/slicing/extract.py new file mode 100644 index 00000000..87645603 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/slicing/extract.py @@ -0,0 +1,105 @@ +"""视频抽帧（切片层的解码入口）。 + +承接视频路线（Issue #35）：i2v 产出的短视频步态真实但为插画质感。本模块只负责 +把视频 bytes 解码成帧序列；选帧（周期 / 一次性）见 :mod:`.loop` / :mod:`.oneshot`， +像素化 / 对齐 / 打包见 :mod:`..postprocess`。抽帧后端（imageio/ffmpeg）函数内惰性， +模块导入零成本、CI 可收集。 +""" + +from __future__ import annotations + +import logging +import os +import tempfile + +from PIL import Image + +logger = logging.getLogger("windup.ai_engine.extract") + +__all__ = ["extract_frames_bytes", "extract_all_frames_bytes"] + + +def extract_frames_bytes(video: bytes, n: int) -> list[Image.Image]: + """从视频 bytes 均匀抽 ``n`` 帧（供后端 strategy 用，provider 返回的是 bytes）。""" + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=True) as f: + f.write(video) + f.flush() + return _extract_frames(f.name, n) + + +def extract_all_frames_bytes(video: bytes, cap: int = 150) -> list[Image.Image]: + """抽视频全部帧（至多 ``cap``，均匀降采样），供周期检测用。""" + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=True) as f: + f.write(video) + f.flush() + return _extract_frames(f.name, cap) + + +def _uniform_indices(total: int, n: int) -> list[int]: + """在 ``total`` 帧里均匀取 ``min(n, total)`` 个下标(含首尾)。""" + m = min(n, total) + return [round(i * (total - 1) / max(1, m - 1)) for i in range(m)] + + +def _frame_count(video_path: str) -> int: + """帧数。先问容器元数据,不可信时退回逐帧计数(计数不保留帧,内存不涨)。 + + 元数据在 14 段真实 i2v 视频上与实际帧数全部一致(2026-08-10 实测),但不同容器/编码 + 的 ``n_frames`` 并非都可靠,所以拿不到正整数就退回计数——多解一遍换一个确定的数, + 比按错的帧数抽出错位的帧划算。 + """ + import imageio.v3 as iio + + try: + shape = iio.improps(video_path, plugin="pyav").shape + if shape and isinstance(shape[0], int) and shape[0] > 0: + return shape[0] + except Exception: + pass + return sum(1 for _ in iio.imiter(video_path, plugin="pyav")) + + +def _extract_frames(video_path: str, n: int) -> list[Image.Image]: + """从视频均匀抽 ``n`` 帧。优先 imageio(流式),回退系统 ffmpeg。 + + **流式而不是一次性读整段**(2026-08-10,机器审 P2):原先走 ``iio.imread`` 会把 + ``(T, H, W, C)`` 整个 materialize 出来。实测 121 帧 720p 的真实 i2v 视频峰值 + 319 MiB,而我们只要其中 8~16 帧;并发 worker 叠加时这是实打实的内存墙。 + 现在峰值≈保留帧数 × 单帧,与视频长度无关。 + """ + try: + import imageio.v3 as iio + + total = _frame_count(video_path) + if total <= 0: + raise RuntimeError("视频无可解码帧") + wanted = set(_uniform_indices(total, n)) + out: list[Image.Image] = [] + for i, frame in enumerate(iio.imiter(video_path, plugin="pyav")): + if i in wanted: + # convert 之后原始 ndarray 就可以被回收;不持有 frame 本身。 + out.append(Image.fromarray(frame).convert("RGBA")) + if len(out) == len(wanted): + break + if out: + return out + except Exception: # noqa: BLE001 - 兜底到 ffmpeg + # 不静默:这个 except 曾把"我们自己算错下标"和"环境里没装 imageio"混为一谈, + # 两者都表现为悄悄换用 ffmpeg 分支、产出看着正常的帧。至少留一条日志。 + logger.warning("imageio 抽帧失败,回退系统 ffmpeg", exc_info=True) + + import glob + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + subprocess.run( + ["ffmpeg", "-y", "-i", video_path, "-vsync", "0", + os.path.join(tmp, "f_%04d.png")], + capture_output=True, check=True, + ) + files = sorted(glob.glob(os.path.join(tmp, "f_*.png"))) + if not files: + raise RuntimeError("抽帧失败:视频无可解码帧") + m = min(n, len(files)) + idx = [round(i * (len(files) - 1) / max(1, m - 1)) for i in range(m)] + return [Image.open(files[i]).convert("RGBA").copy() for i in idx] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/slicing/loop.py b/backend/packages/ai_engine/src/windup_ai_engine/slicing/loop.py new file mode 100644 index 00000000..26a5b586 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/slicing/loop.py @@ -0,0 +1,174 @@ +"""循环闭合(最后一公里之一,Issue #21)—— 从 i2v 密集帧里抽正好一个步态周期,做无缝 loop。 + +i2v 的 5s 视频里含 ~2-3 个步态周期,均匀抽 N 帧跨多个周期 → 首尾接缝跳。做法: +帧自相似检测周期(灰度小图,frame[i] 与 frame[i+p] 差最小的 p = 一个周期), +再在一个周期内均匀取 N 帧 → frame[N-1] 的下一拍≈frame[0],循环自然闭合。 +纯 numpy / PIL,零 API。 +""" +from __future__ import annotations + +import numpy as np +from PIL import Image + +from ._frames import SMALL as _SMALL +from ._frames import gray as _gray + +__all__ = ["find_period", "pick_cycle"] + + +def _deskew(gs: list[np.ndarray]) -> list[np.ndarray]: + """消掉角色的整体水平平移再比对。i2v 里角色会横着挪(实测走位达画宽 18%),而下游 + :func:`postprocess.pack.align_bottom_center` 本来就会逐帧重新居中 —— 不消平移的话, + d(p) 被"挪了多远"主导、随 p 单调上升,真周期的凹陷被压平(骷髅走路真周期 56 完全消失, + 只剩 22 的假凹陷)。做法:中值背景差取主体列范围,把质心 roll 到画面中心。""" + a = np.stack(gs) + d = np.abs(a - np.median(a, axis=0)) + thr = max(4.0, float(np.percentile(d, 99)) * 0.25) + out = [] + for g, di in zip(gs, d): + cols = (di > thr).any(0) + cx = float(np.where(cols)[0].mean()) if cols.any() else _SMALL / 2 + out.append(np.roll(g, int(round(_SMALL / 2 - cx)), axis=1)) + return out + + +def _dmat(gs: list[np.ndarray]) -> np.ndarray: + """全帧对距离矩阵(48x48 灰度平均绝对差),后续所有判据共用,只算一次。""" + flat = np.stack(gs).reshape(len(gs), -1) + return np.stack([np.abs(flat - flat[i]).mean(1) for i in range(len(gs))]) + + +def _curve(M: np.ndarray, pmin: int, pmax: int) -> dict[int, float]: + n = len(M) + return {p: float(np.mean([M[i, i + p] for i in range(n - p)])) for p in range(pmin, pmax + 1)} + + +def _prominent_period(curve: dict[int, float], scale: float) -> tuple[int, float] | None: + """只认"内部局部极小 + 凹陷够深"的 p。曲线单调(无周期)时返回 None,而不是交出边界值。""" + ps = sorted(curve) + best = None + for j in range(1, len(ps) - 1): + p = ps[j] + if not (curve[p] <= curve[ps[j - 1]] and curve[p] <= curve[ps[j + 1]]): + continue + w = max(3, p // 2) + lo = [curve[q] for q in ps if p - w <= q < p] + hi = [curve[q] for q in ps if p < q <= p + w] + if not lo or not hi: + continue + prom = (min(max(lo), max(hi)) - curve[p]) / max(scale, 1e-6) + if best is None or prom > best[1]: + best = (p, prom) + return best + + +def find_period(frames: list[Image.Image], pmin: int | None = None, pmax: int | None = None) -> int: + """自相似求步态周期(帧数)。frame[i] 与 frame[i+p] 平均差最小的 p。""" + n = len(frames) + gs = _gray(frames) + pmin = pmin or max(4, n // 6) + pmax = pmax or max(pmin + 1, n // 2) + best_p, best_d = pmin, float("inf") + for p in range(pmin, pmax + 1): + d = float(np.mean([np.abs(gs[i] - gs[i + p]).mean() for i in range(n - p)])) + if d < best_d: + best_d, best_p = d, p + return best_p + + +def _offsets(P: int, n: int) -> list[int]: + return [round(k * P / n) for k in range(n)] + + +def pick_cycle(frames: list[Image.Image], n: int) -> list[Image.Image]: + """从密集帧里抽正好一个步态周期的 N 帧(无缝 loop)。返回长度恒等于 ``n``。 + + 周期检测的三个坑(实测 5 段真 i2v 视频): + 1. d(p) 会被"角色整体平移 + 画质漂移"抬成单调上升 —— 直接取 argmin 会滑到搜索窗边界 + 交出**假周期**(走路视频: p 恒等于 pmin)。故只认有足够凹陷深度的内部局部极小, + 测不到就判"无周期",退化成全片均匀取(不硬闭环) —— 硬闭环反而制造接缝。 + 2. 搜索窗要覆盖真周期: 上界 n//2 会把 5s 里只有 ~2 个周期的待机挡在窗外(实测真周期 62 + > pmax 60,于是取到边界值);下界要 >= 目标帧数 n,否则 round(k*p/n) 直接产出重复帧。 + 3. 谐波: 平移偏置让 d(p) 偏爱短 lag,常选中真周期的 1/2(骷髅走路: 取到 22,真周期 ~56), + 半周期闭环 = 末帧接回首帧时左右腿瞬间互换 = 肉眼可见的"跳一下"。故在 p 的整数倍里 + 按**归一化接缝**(末→首 差 / 组内相邻差均值)复选,并保证取样索引互不重复。 + 不要在这里接 :mod:`.quality` 的死帧判据 —— 2026-08-07 消融实测(4 段真 i2v)证否: + ① 用 ``active_span`` 先掐头尾冻结段:``_deskew`` 已经解决了"曲线被不动的帧压平"这个 + 问题,再掐只是缩小 i0 的搜索空间、丢掉更优相位起点(奔跑 seam 0.81→2.00); + ② 取样后按 ``dead_frame_mask`` 就近避让死帧:i2v 死帧占比常达一半(24fps 容器隔帧复制, + 实测 59/121 与 63/121),避让会系统性打乱相位均匀性,反而选中更多死帧(1→3)。 + 候选评分里的 ``a < 0.5 * scale`` 已经排掉"几乎不动"的窗口,够用。 + quality 不进选帧(它另有一份出参职责,见 :mod:`.quality`)。 + + **已知缺口(2026-08-09 记,未修)**:本函数有四条返回路径,其中三条是 return 帧列表, + 调用方**分不清走了哪条**: + 1. ``total <= n`` 原样返回(源帧比要的还少,根本没选); + 2. 测不到可信周期 → 全片均匀取(**降级**,不闭环); + 3. 候选全被否 → 全片均匀取(**降级**,同上); + 4. 正常闭环。 + 2 与 3 的补救方式不同(2 多半是母版/动作幅度问题,该换母版;3 多半是视频里周期数 + 不够,该加长视频),但今天都表现为"一组看起来正常的帧"。**降级对交付物的后果**是可测 + 的 —— ``ports.ActionQuality.loop_seam`` 在交付帧上量归一化接缝,降级通常表现为接缝 + 偏大;但**降级的原因**测不出来。没有顺手把状态塞进返回值,是因为那要改本函数的返回 + 形状、波及所有调用方,而今天还没有任何调用方会依据"原因"改变行为。等真有调用方要按 + 原因给不同提示时,再让本函数返回 ``(frames, reason)``。 + """ + # n<=0 没有合法语义(要 0 帧的动画不存在),且两条出路都是坏的(2026-08-10 实测): + # 检出周期时 `_offsets(P, 0)` 交出空 offsets,一路走到 `M[idx[-1], idx[0]]` 抛 IndexError; + # 测不到周期时(单调曲线)直接静默返回 [] —— 后者更危险,故在入口显式拒绝。 + # (机器审说这里除零并不准确:`range(n)` 为空,`k*P/n` 根本没被求值。) + if n <= 0: + raise ValueError(f"n 必须 >= 1,收到 {n}") + total = len(frames) + if total < n: + # 源帧不够就报错,不再原样返回:长度不足且不报错,下游 frame_durations 按实际长度现算, + # 帧数与时长表自洽,server 看不出异常,用户拿到的是一段没走完的循环。 + raise ValueError(f"源帧不足:请求 {n} 帧,只有 {total} 帧") + if total == n: + return frames + M = _dmat(_deskew(_gray(frames))) + if n == 1: + # 单帧"循环"没有接缝也没有相位,下面整套周期/接缝机制全部失效(实测 n=1 时窗口内相邻差 + # 是空均值 = nan,`a < 0.5*scale` 与接缝评分双双被 nan 短路,靠比较运算的意外结果才 + # 返回 frames[0])。显式取 medoid:与全片平均姿态最近的一帧 = 循环停留最久的相位, + # 比 frames[0](i2v 的首帧是母版静立姿,单看读不出"在走")更能代表这个循环。 + return [frames[int(np.argmin(M.mean(1)))]] + adj = np.array([M[i, i + 1] for i in range(total - 1)]) + scale = float(np.median(adj)) + + pmin = max(6, min(n, total // 6)) + pmax = min(total - 3, max(pmin + 2, int(total * 0.6))) + got = _prominent_period(_curve(M, pmin, pmax), scale) + if got is None or got[1] < 0.25: # 测不到可信周期 → 不硬闭环 + idx = [round(k * (total - 1) / n) for k in range(n)] + return [frames[i] for i in idx] + + p = got[0] + cands = [] + for k in range(1, 4): # 在基周期的整数倍里复选 + P = k * p + if P > total - 2: + break + offs = _offsets(P, n) + if len(set(offs)) < n: # 该倍数取不出 n 个不同相位 + continue + best = None + for i0 in range(total - P): + idx = [i0 + o for o in offs] + a = float(np.mean([M[idx[j], idx[j + 1]] for j in range(n - 1)])) + if a < 0.5 * scale: # 窗口几乎不动(i2v 尾部常停顿)→ 弃 + continue + score = M[idx[-1], idx[0]] / max(a, 1e-6) # 归一化接缝 + if best is None or score < best[0]: + best = (score, idx) + if best: + cands.append((k, best[0], best[1])) + if not cands: + idx = [round(k * (total - 1) / n) for k in range(n)] + return [frames[i] for i in idx] + # 倍数越大 = 一个 loop 里塞进越多周期 = 每周期帧数越少(动作变糙),故**优先最小倍数**: + # 取第一个"已经闭合"的倍数(归一化接缝 <= 1.2,即末→首的跳幅不超过一个正常帧间步长), + # 都不闭合才退而取最优 —— 这一条专治"取到半周期 → 接缝处左右腿瞬间互换"。 + ok = [c for c in cands if c[1] <= 1.2] + pick = ok[0] if ok else min(cands, key=lambda c: c[1]) + return [frames[i] for i in pick[2]] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/slicing/oneshot.py b/backend/packages/ai_engine/src/windup_ai_engine/slicing/oneshot.py new file mode 100644 index 00000000..dcbf1507 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/slicing/oneshot.py @@ -0,0 +1,246 @@ +"""一次性动作(jump / attack / hit)的抽帧:裁动作起止 + 按状态切段。 + +与循环类(idle/walk/run)的根本差别: +- 循环类用 :mod:`.loop` 找步态周期抽单周期闭环;一次性动作**不能闭环** —— 首尾姿态不同, + 强行闭环会把落地帧接回蓄力帧,读起来是抽搐。 +- i2v 出的 5s 视频里,真正的动作往往只占中间一段(前后是静止的起手/终态保持),直接均匀 + 抽帧会浪费一半帧在不动的地方 → 需要先**裁到动作发生的区间**。 +- jump 还要进一步**按状态切段**(蓄力/上升/顶点/下降/落地),因为引擎里悬空时长由物理 + 决定、上升中可被打断,必须能分段播放。 + +纯 numpy / PIL,零 API。 +""" + +from __future__ import annotations + +import numpy as np +from PIL import Image + +from windup_ai_engine._subject import subject_mask as _subject_mask + +__all__ = [ + "find_motion_span", + "first_action_end", + "pick_oneshot", + "split_jump_phases", + "foot_line_series", +] + + +_KINDS = ("swing", "airborne") + + +def _check_kind(kind: str) -> None: + """未知 ``kind`` 直接报错,不静默回落到 ``swing``。 + + 两个判据的物理不同(脚线回地 vs 能量跌破),拼错一个字母(``"airbourne"``)若被当成 + ``swing`` 处理,跳跃会按能量判据裁 —— 出的是"看起来成功"的错区间(实测:顶点悬停处 + 能量安静,动作被截在半空),而这类错误在序列帧里很难回溯到 kind 拼错上。 + """ + if kind not in _KINDS: + raise ValueError(f"kind 只能是 'swing' 或 'airborne',收到 {kind!r}") + + +def _frame_energy(frames: list[Image.Image], size: int = 64) -> np.ndarray: + """逐帧与前一帧的差异强度(灰度小图),长度 = len(frames)-1。""" + gs = [np.asarray(f.convert("L").resize((size, size)), dtype=np.float32) for f in frames] + return np.array([np.abs(gs[i + 1] - gs[i]).mean() for i in range(len(gs) - 1)]) + + +def find_motion_span(frames: list[Image.Image], rel_thr: float = 0.25) -> tuple[int, int]: + """定位"动作真正发生"的帧区间 ``[start, end]``(含端点)。 + + 以帧间差异强度超过峰值 ``rel_thr`` 倍的最早/最晚位置为界,并各留一帧余量。 + 静止的起手与终态保持会被裁掉。 + """ + if len(frames) < 3: + return 0, len(frames) - 1 + e = _frame_energy(frames) + peak = float(e.max()) + if peak <= 1e-6: + return 0, len(frames) - 1 + active = np.flatnonzero(e >= peak * rel_thr) + if not len(active): + return 0, len(frames) - 1 + start = max(0, int(active[0]) - 1) + end = min(len(frames) - 1, int(active[-1]) + 2) + return start, end + + +def _airborne_end(frames: list[Image.Image], start: int, end: int, tol: float = 6.0) -> int: + """腾空类(jump)的结束:脚线越过最高点后**首次回到地面**。 + + 几何信号,明确无歧义 —— 比任何"能量安静"判据都稳。 + """ + y = foot_line_series(frames[start : end + 1]) + if len(y) < 4: + return end + apex = int(np.argmin(y)) + ground = float(np.median([y[0], y[-1]])) + back = np.flatnonzero(y[apex:] >= ground - tol) + return min(end, start + apex + int(back[0]) + 2) if len(back) else end + + +def _swing_end(frames: list[Image.Image], start: int, end: int, + drop_ratio: float = 0.35, recover: int = 2) -> int: + """挥击类(attack/hit)的结束:能量越过峰值后**首次跌到峰值的 ``drop_ratio``**,再留收势余量。 + + 挥击是"蓄力 → 峰值 → 收势"的单峰结构,收势很短,故用"跌破比例 + 固定余量"即可; + 不要求长时间静止 —— 实测挥砍收势段的能量并不干净(视频压缩噪点),等不到静止平台。 + """ + e = _frame_energy(frames[start : end + 1]) + if len(e) < 4: + return end + peak_i = int(np.argmax(e)) + thr = float(e.max()) * drop_ratio + for i in range(peak_i + 1, len(e)): + if e[i] < thr: + return min(end, start + i + recover) + return end + + +def first_action_end( + frames: list[Image.Image], start: int, end: int, kind: str = "swing" +) -> int: + """在 ``[start, end]`` 内找**第一次**动作的结束帧,按动作物理分流。 + + i2v 常在 5s 里把一次性动作**复读第二遍**(实测:提示词写了 "ONCE",兽人跳了两次、 + 挥砍也挥了两次),不裁会把两次动作压进一套序列帧。 + + 不同动作的"结束"信号本质不同,**一个通用判据管不了两种**(实测踩过): + - ``kind="airborne"``(jump):脚线回到地面 —— 几何、无歧义。 + - ``kind="swing"``(attack/hit):能量跌破峰值比例 + 收势余量。 + + 三个已验证无效的通用解法(别再试):①只看"帧间安静" → 在跳跃**顶点悬停**处误触发, + 把动作截在半空;②要求静止段足够长 → 挥砍收势并不干净(压缩噪点),等不到,完全不裁; + ③找"回到起始姿态"的谷底 → 收势姿态(戒备)与起始姿态(蓄力)不同,回不到低位。 + """ + _check_kind(kind) # 放在早返回之前:短区间也不能放过拼错的 kind + if end - start < 4: + return end + return (_airborne_end if kind == "airborne" else _swing_end)(frames, start, end) + + +def _key_pose(span: list[Image.Image], kind: str) -> int: + """区间内最能代表这次动作的单帧下标(关键姿势)。 + + 只取一帧时不能取区间首帧或中点: + - 首帧是蓄力起手,和待机几乎一个样,单看认不出这是攻击还是跳跃;末帧是收势/落地,同理。 + - 中点也不行:动作区间前后不对称(蓄力长、收势短 —— :func:`_swing_end` 只留 2 帧余量), + 中点会落进蓄力段。 + 故取"关键姿势":判据与 :func:`first_action_end` 同源,不引入新参数 —— + ``airborne`` 取脚线最高(顶点),``swing`` 取能量峰。能量是**帧间**差(长度 len-1), + 峰值下标 i 表示 i→i+1 这一跳变化最大,故取 i+1 = 刚完成最快一段位移的那一帧(命中瞬间)。 + """ + if len(span) < 2: + # 单帧区间:答案唯一(就那一帧),没有歧义,不必炸。当前 pick_oneshot 恒给 >= 2 帧 + # (end 至少 start+1),但那是上游三个启发式判据合出来的保证、不是本函数签名的保证, + # 故留这一行 —— 否则 _frame_energy 会交出空数组、argmax 抛一个看不懂的 numpy 错。 + return 0 + if kind == "airborne": + return int(np.argmin(foot_line_series(span))) + return min(len(span) - 1, int(np.argmax(_frame_energy(span))) + 1) + + +def _widen_span(start: int, end: int, n: int, total: int) -> tuple[int, int]: + """区间不足 n 帧时把窗口放宽回来,保证能取出 n 个**互不相同**的源帧。 + + 调用处已保证 ``total > n``,即源帧数是够的 —— 区间短只是我们自己的裁剪判据收得紧 + (2026-08-10 实测:14 帧输入裁到 9 帧区间,请求 12 帧只回 9 帧)。此时既不该报错(源帧够), + 更不该静默少给帧:下游 ``frame_durations(action, len(frames))`` 按实际长度现算时长,帧数与 + 时长表自洽,server 看不出异常,用户拿到的是一段步子没走完的动作。故按缺口对称放宽, + 宁可带上几帧起手/收势的静止帧,也要给足请求的帧数。 + + 动作贴在视频尾部时右边长不动,缺口必须退回左边补(最后一行),否则窗口仍不足 n 帧、 + 只能靠重复帧凑数 —— 长度对、内容卡顿,又是一种"看起来成功"。 + """ + deficit = n - (end - start + 1) + if deficit <= 0: + return start, end + left = min(start, (deficit + 1) // 2) # 先往左补一半,左边不够就全从右边补 + start -= left + end = min(total - 1, end + deficit - left) + return max(0, end - n + 1), end # 右边撞到尾部时把缺口退回左边 + + +def pick_oneshot( + frames: list[Image.Image], n: int, first_only: bool = True, kind: str = "swing" +) -> list[Image.Image]: + """一次性动作抽 ``n`` 帧:裁到动作区间 → 只留第一次动作 → 区间内均匀取(不闭环)。 + + ``first_only`` 默认开:防 i2v 在 5s 内复读第二遍动作被一起抽进来。 + ``kind``:``"airborne"``(jump,按脚线回地判结束)或 ``"swing"``(attack/hit,按能量跌破判)。 + + 返回长度**恒等于** ``n``;源帧不够 n 帧则报错,不静默少给。 + """ + _check_kind(kind) # first_only=False 时不走 first_action_end,这里兜住 + if n <= 0: + # 2026-08-10 实测:n<=0 原本静默返回 [](range(n) 为空,连除零都不报),"成功"地交出零帧。 + raise ValueError(f"n 必须 >= 1,收到 {n}") + if len(frames) < n: + raise ValueError(f"源帧不足:请求 {n} 帧,只有 {len(frames)} 帧") + if len(frames) == n: + return frames + start, end = find_motion_span(frames) + if first_only: + end = max(start + 1, first_action_end(frames, start, end, kind=kind)) + start, end = _widen_span(start, end, n, len(frames)) + span = frames[start : end + 1] + if n == 1: # n=1 撞下面的 /(n-1) 除零(机器审 P1,2026-08-10 复现) + return [span[_key_pose(span, kind)]] + idx = [round(i * (len(span) - 1) / (n - 1)) for i in range(n)] + return [span[i] for i in idx] + + +def _subject_rows(frame: Image.Image, alpha_thr: int = 128, bg_tol: int = 60) -> np.ndarray: + """主体所在的行下标。判据本身在 :mod:`.._subject`(与母版预检共用同一个主体定义)。""" + return np.where(_subject_mask(frame, alpha_thr, bg_tol))[0] + + +def foot_line_series(frames: list[Image.Image], alpha_thr: int = 128) -> np.ndarray: + """逐帧主体**底边** y 坐标(脚线)。跳跃时脚线先降(蹲)、再升(腾空)、再落回。""" + out = [] + for f in frames: + ys = _subject_rows(f, alpha_thr) + out.append(float(ys.max()) if len(ys) else np.nan) + arr = np.array(out, dtype=np.float32) + if np.isnan(arr).any(): # 空帧用邻近值补 + idx = np.arange(len(arr)) + good = ~np.isnan(arr) + if good.any(): + arr = np.interp(idx, idx[good], arr[good]) + else: + arr = np.zeros_like(arr) + return arr + + +def split_jump_phases(frames: list[Image.Image]) -> dict[str, list[int]]: + """按脚线轨迹把跳跃切成 crouch / rise / apex / fall / land 五段,返回每段的帧下标。 + + 判据:脚线 y 越小 = 人越高。最高点(y 最小)即 apex;起跳前脚线最低(蹲)处为 crouch + 结束;之后到 apex 为 rise,apex 之后到脚线回到地面高度为 fall,余下为 land。 + 只依赖几何,不依赖模型。 + """ + n = len(frames) + if n < 5: + return {"rise": list(range(n))} + y = foot_line_series(frames) + apex = int(np.argmin(y)) # 最高点 + ground = float(np.median([y[0], y[-1]])) # 地面脚线 + # 起跳点:apex 之前脚线最低(数值最大 = 蹲得最深)的位置 + takeoff = int(np.argmax(y[: max(1, apex)])) if apex > 0 else 0 + # 落地点:apex 之后脚线首次回到地面附近 + after = y[apex:] + back = np.flatnonzero(after >= ground - 2) + landing = apex + int(back[0]) if len(back) else n - 1 + + apex_lo = max(takeoff + 1, apex - 1) + apex_hi = min(landing - 1, apex + 1) + phases = { + "crouch": list(range(0, takeoff + 1)), + "rise": list(range(takeoff + 1, apex_lo)), + "apex": list(range(apex_lo, apex_hi + 1)), + "fall": list(range(apex_hi + 1, landing)), + "land": list(range(landing, n)), + } + return {k: v for k, v in phases.items() if v} diff --git a/backend/packages/ai_engine/src/windup_ai_engine/slicing/quality.py b/backend/packages/ai_engine/src/windup_ai_engine/slicing/quality.py new file mode 100644 index 00000000..734274ec --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/slicing/quality.py @@ -0,0 +1,148 @@ +"""帧质量诊断:死帧(重复/冻结)与坏帧(糊/伪影)的判据。 + +与 :mod:`.loop` 的分工:loop 负责选帧,本模块只负责"这帧是什么成色"。 +2026-08-05 实测(6 个真 i2v 视频):**没有一帧糊帧**,但死帧极多——24fps 容器里 +隔一帧就是复制帧(奔跑视频奇偶帧差比 22.5x),有效内容帧率只有 ~11-14fps; +且普遍有起步冻结(头部)或动作衰减停住(尾部)。所以"坏帧"与"死帧"必须分开判、 +分开统计——只判一型会漏掉一半。 +""" +from __future__ import annotations + +import numpy as np + +from ._frames import gray as _gray + +__all__ = ["active_span", "blur_ratio", "dead_frame_indices", "dead_frame_mask", + "frame_deltas", "loop_seam", "motion_scale"] + + +def frame_deltas(frames) -> np.ndarray: + """d[i] = |f_i - f_{i-1}| 均值，d[0]=0。小图（48x48 灰度），CPU 便宜。""" + gs = _gray(frames) + return np.array([0.0] + [float(np.abs(gs[i] - gs[i - 1]).mean()) for i in range(1, len(gs))]) + + +def dead_frame_mask(frames, ratio: float = 0.35, floor: float = 0.25) -> np.ndarray: + """死帧 = 相对前一帧几乎没有新内容。两型必须都判，缺一漏一半： + + A 型「隔帧死」: d[i] < ratio * max(d[i-1], d[i+1]) + i2v 常见"有效帧率减半"——24fps 容器里隔一帧就是复制帧。只用全局阈值抓不到， + 因为半数帧是死帧时 median 本身落在死帧堆里（实测 run 奇偶比 9.9x 却报 0 死帧）。 + B 型「持续冻结」: d[i] < floor * p75(d) + 视频头部的 i2v 起步冻结、尾部的动作衰减停住。只用 A 型抓不到， + 因为连续冻结段里邻居同样低，比值≈1（实测 attack 尾部 9 帧全漏）。 + """ + d = frame_deltas(frames) if not isinstance(frames, np.ndarray) else frames + n = len(d) + p75 = float(np.percentile(d[1:], 75)) if n > 1 else 0.0 + m = np.zeros(n, dtype=bool) + for i in range(1, n): + nb = [d[j] for j in (i - 1, i + 1) if 1 <= j < n] + if nb and d[i] < ratio * max(nb): + m[i] = True + if d[i] < floor * p75: + m[i] = True + return m + + +def dead_frame_indices(frames) -> tuple[int, ...]: + """死帧下标。:func:`dead_frame_mask` 的出参形态转换 —— 掩码是算的时候好用的形态, + 跨出 ai_engine 的契约(``ports.ActionQuality``)要的是"哪几帧",不该让调用方拿着 + 一个 numpy 掩码去自己 argwhere。""" + return tuple(int(i) for i in np.flatnonzero(dead_frame_mask(frames))) + + +def motion_scale(frames) -> float: + """相邻帧平均差异的**绝对**尺度(48×48 灰度)。0.0 = 这些帧逐像素完全一样。 + + 为什么与 :func:`dead_frame_mask` 并存、而不是从它推导:后者两条判据 + (``d[i] < ratio*max(邻居)`` 与 ``d[i] < floor*p75``)**都是相对的**,整段完全 + 冻结时 d 全为 0,两条不等式变成 ``0 < 0``,一条都不成立 —— **一帧死帧都报不出** + (2026-08-09 用全同帧序列实测:12 帧全同,死帧数 0)。相对判据天生看不见"整体 + 没动",绝对尺度必须单独给一个。 + """ + d = frame_deltas(frames) + return float(d[1:].mean()) if len(d) > 1 else 0.0 + + +def loop_seam(frames) -> float | None: + """末帧接回首帧的跳幅 ÷ 相邻帧平均步长;整段静止(分母为 0)返回 ``None``。 + + 与 :func:`.loop.pick_cycle` 选帧时的归一化接缝同式,但**测的对象不同**:pick_cycle + 在抠图 / 像素化 / 脚线对齐**之前**的密集帧上打分,而用户看到的是这三步之后的帧, + 这三步都会改动像素。要描述交付物就得在交付物上量。 + + 不套 :func:`.loop._deskew`:交付帧已被 ``align_bottom_center`` 逐帧居中,整体平移 + 早消掉了,再按差分质心对一次只是引入第二套居中口径(两套口径不一致正是本仓反复 + 踩的那类静默分歧)。 + + 分母为 0 时返回 None 而不是 0.0 —— 0.0 会被读成"完美闭环",而真相是"没有可比的 + 步长,这个数不可读"。 + """ + gs = _gray(frames) + if len(gs) < 2: + return None + step = float(np.mean([np.abs(gs[i + 1] - gs[i]).mean() for i in range(len(gs) - 1)])) + if step <= 0.0: + return None + return float(np.abs(gs[-1] - gs[0]).mean() / step) + + +def active_span(frames, floor: float = 0.25, min_run: int = 3) -> tuple[int, int]: + """掐掉头尾的**持续**冻结段，返回 [s, e]（闭区间）。中间的隔帧死不动。""" + d = frame_deltas(frames) + n = len(d) + p75 = float(np.percentile(d[1:], 75)) if n > 1 else 0.0 + low = d < floor * p75 + s, e = 0, n - 1 + r = 0 + for i in range(1, n): # 头部 + if low[i]: + r += 1 + else: + break + if r >= min_run: + s = r + r = 0 + for i in range(n - 1, 0, -1): # 尾部 + if low[i]: + r += 1 + else: + break + if r >= min_run: + e = n - 1 - r + if e - s < 4: # 掐过头就放弃 + return 0, n - 1 + return s, e + + +def blur_ratio(frames, ps: int = 32) -> np.ndarray: + """逐帧「静止区清晰度 / 前后帧同区清晰度」。<1 = 这帧自己糊了，与动作快慢无关。""" + def _pm(a): + h, w = a.shape + H, W = max(ps, h // ps * ps), max(ps, w // ps * ps) + a = a[:H, :W] + return a.reshape(H // ps, ps, W // ps, ps).mean(axis=(1, 3)) + + def _ag(g): + gx = np.zeros_like(g) + gy = np.zeros_like(g) + gx[:, 1:-1] = np.abs(g[:, 2:] - g[:, :-2]) * .5 + gy[1:-1, :] = np.abs(g[2:, :] - g[:-2, :]) * .5 + return np.maximum(gx, gy) + + gs = [np.asarray(f.convert("L"), np.float32) for f in frames] + sharp = np.stack([_pm(_ag(g)) for g in gs]) + out = np.ones(len(gs), np.float32) + for i in range(1, len(gs) - 1): + mv = np.maximum(_pm(np.abs(gs[i] - gs[i - 1])), _pm(np.abs(gs[i] - gs[i + 1]))) + ref = .5 * (sharp[i - 1] + sharp[i + 1]) + m = (mv < 2.5) & (ref > 3.0) + if m.sum() < 4: + cand = ref > 3.0 + if cand.sum() < 4: + continue + k = max(4, int(cand.sum() * .25)) + m = cand & (mv <= np.sort(mv[cand])[:k].max()) + out[i] = float(np.median(sharp[i][m] / np.maximum(ref[m], 1e-6))) + return out diff --git a/backend/packages/ai_engine/src/windup_ai_engine/strategy/__init__.py b/backend/packages/ai_engine/src/windup_ai_engine/strategy/__init__.py new file mode 100644 index 00000000..8435572e --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/strategy/__init__.py @@ -0,0 +1,12 @@ +"""strategy:动作 → 生成路线分流(ROUTE_MATRIX)+ 三条 DerivationStrategy。""" + +from .base import CYCLIC_ACTIONS, ROUTE_MATRIX, DerivationStrategy +from .concrete import PerFrameStrategy, VideoFrameStrategy + +__all__ = [ + "ROUTE_MATRIX", + "CYCLIC_ACTIONS", + "DerivationStrategy", + "VideoFrameStrategy", + "PerFrameStrategy", +] diff --git a/backend/packages/ai_engine/src/windup_ai_engine/strategy/base.py b/backend/packages/ai_engine/src/windup_ai_engine/strategy/base.py new file mode 100644 index 00000000..0a1b1c00 --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/strategy/base.py @@ -0,0 +1,66 @@ +"""DerivationStrategy —— 按动作类型分流到生成路线(本营实测挣得的核心架构决策)。 + +分流依据(有实测证据,非拍脑袋,详见关联 Issue #35 的工程文档): + - 步态位移(walk / run):逐帧独立生成锁不住"哪条腿在前" → 踢踏舞; + 必须走视频 i2v(视频模型天生连贯、腿自然交替)。 + - 动作爆发(attack)与跳跃(jump):同走视频 i2v。但它们是**一次性动作**,抽帧不闭环 + (见本模块 CYCLIC_ACTIONS);jump 还要按状态切段供引擎分段播放。 + - 受击等离散姿势(hit):逐帧图生图(单帧可编辑价值高,无连续步态)。 + - 待机(idle):逐帧生成只抖不呼吸 → 程序化局部呼吸 Idle-B。 + +ROUTE_MATRIX 是人主导的架构契约,改它=改产线,要有实测支撑。 +""" +from __future__ import annotations + +from abc import ABC, abstractmethod + +from windup_common.models import ActionSpec, ActionType, CharacterCard, GenRoute + +from windup_ai_engine.ports import ProgressPort + +# 动作类型 → 生成路线(架构决策,写死为契约) +ROUTE_MATRIX: dict[ActionType, GenRoute] = { + ActionType.WALK: GenRoute.VIDEO_I2V, + ActionType.RUN: GenRoute.VIDEO_I2V, + ActionType.JUMP: GenRoute.VIDEO_I2V, + ActionType.ATTACK: GenRoute.VIDEO_I2V, + ActionType.HIT: GenRoute.PER_FRAME, + # idle 走 i2v(build_idle_prompt:躯干缓慢起伏呼吸)。 + # **2026-08-07 定案**:#53 原设计的 ¥0 程序化 Idle-B(局部网格呼吸)放弃 —— 做不出 + # 可用效果,idle 认这份 i2v 的钱。GenRoute.PROC_IDLE 与 ProcIdleStrategy 已一并移除。 + ActionType.IDLE: GenRoute.VIDEO_I2V, +} + +# 循环类动作:抽单步态周期闭环。一次性动作**不能闭环**(首尾姿态不同,强行闭环会把 +# 落地帧接回蓄力帧=抽搐),走"裁动作区间 + 区间内均匀取"。 +# 与 ROUTE_MATRIX 并排放在 base 而不是留在 concrete:它同样是「动作类型 → 产线行为」的 +# 契约,且现在有两个消费方 —— strategy.concrete 用它选抽帧方式,impl.CharacterGenerator +# 用它决定交付成色里的 loop_seam 该不该测(不闭环的动作没有"接缝"可言)。放在 concrete +# 会让 generator 为了问一句"这动作循环吗"去 import 一条具体路线的实现。 +CYCLIC_ACTIONS: frozenset[ActionType] = frozenset( + {ActionType.IDLE, ActionType.WALK, ActionType.RUN} +) + +# 本矩阵的形状本身有个已知边界,记录在此以免后来者按错误前提扩展: +# 它是「动作类型 → 路线」的一对一映射,隐含前提是"路线由动作的物理性质唯一决定"。 +# 该前提对逐帧 / 视频两条路线成立(有无连续步态是动作固有属性),但对渲染出帧路线不成立 +# —— 同一个 walk 既可走 i2v 也可走渲染,选哪条取决于"该角色有没有 3D 模型",那是 server +# 才知道的事。接入第三条路线前须先定「路线选择由谁决定」,并可能要把本矩阵改成 +# 「动作类型 → 可选路线集合」+ 一个选择器。Refs 1024XEngineer/Windup#81 #122。 + + +class DerivationStrategy(ABC): + """一条生成路线的骨架:母版 → 对齐前的角色帧序列。""" + + route: GenRoute + + @abstractmethod + def derive( + self, + card: CharacterCard, + action: ActionSpec, + master: bytes, + progress: ProgressPort, + ) -> list[bytes]: + """从母版 bytes 产出对齐前的角色帧(RGBA PNG bytes 列表)。""" + raise NotImplementedError diff --git a/backend/packages/ai_engine/src/windup_ai_engine/strategy/concrete.py b/backend/packages/ai_engine/src/windup_ai_engine/strategy/concrete.py new file mode 100644 index 00000000..eff5521c --- /dev/null +++ b/backend/packages/ai_engine/src/windup_ai_engine/strategy/concrete.py @@ -0,0 +1,148 @@ +"""三条 DerivationStrategy。 + +- VideoFrameStrategy：**已迁入 windup-pipeline 实测通路**（walk 主链，2026-07-27 验证）。 +- PerFrameStrategy：**未实现**，调用即抛 NotImplementedError（见 #53）。不返回空帧—— + 空帧会伪装成一次成功的生成流到 server 落库，用户看到的是一组裂图。 + +VideoFrameStrategy 实测通路：严格侧面母版 → kling i2v(v2-5-turbo) → 抽单循环 N 帧 → +matte 抠图 → 像素化。返回对齐前的 RGBA PNG 帧（对齐 / 打包在 CharacterGenerator 最后一公里）。 +""" +from __future__ import annotations + + +import numpy as np + +from windup_common.models import ActionSpec, ActionType, CharacterCard, GenRoute, Stylize +from windup_framework.providers import ImageProvider, MatteProvider, VideoProvider + +from windup_ai_engine._imgio import from_png as _img +from windup_ai_engine._imgio import to_png as _png +from windup_ai_engine.master_prep import prepare_master +from windup_ai_engine.ports import ProgressPort +from windup_ai_engine.postprocess import master_pixel_spec, pixelate_frames +from windup_ai_engine.slicing import extract_all_frames_bytes, pick_cycle, pick_oneshot +from windup_ai_engine.prompt import ( + build_attack_prompt, + build_idle_prompt, + build_jump_prompt, + build_walk_prompt, +) +from windup_ai_engine.strategy.base import CYCLIC_ACTIONS, DerivationStrategy + + +class VideoFrameStrategy(DerivationStrategy): + """视频路线：母版 → i2v → 抽帧 → 抠图 → 像素化。 + + 覆盖循环类(walk/run)与一次性类(jump/attack)——按 :data:`CYCLIC_ACTIONS` 分流抽帧方式。 + 硬前提：**提示词朝向必须与母版一致**(side/front)；给正面母版喂侧走词会让模型靠转身 + 调和图文矛盾(实测 #35)。 + """ + + route = GenRoute.VIDEO_I2V + + def __init__(self, video: VideoProvider, matte: MatteProvider) -> None: + self._video = video + self._matte = matte + + def _build_prompt(self, action: ActionSpec) -> str: + """按动作类型选提示词;朝向随 ActionSpec.facing。""" + builders = { + ActionType.JUMP: build_jump_prompt, + ActionType.IDLE: build_idle_prompt, + ActionType.ATTACK: build_attack_prompt, + } + build = builders.get(action.action, build_walk_prompt) + return build(facing=action.facing) + + def derive( + self, + card: CharacterCard, + action: ActionSpec, + master: bytes, + progress: ProgressPort, + ) -> list[bytes]: + # 帧数直接读契约字段:缺省值已收进 ActionSpec(DEFAULT_N_FRAMES),不再由本层 + # 用 `or 8` 兜底 —— 那等于把契约的缺省值写在实现里,换条 strategy 就换个默认值。 + n = action.n_frames + # 进度文案里的枚举一律取 .value:Python 3.11+ 改了 str-mixin 枚举的 __format__, + # f"{action.action}" 现在给的是 "ActionType.WALK" 而不是 "walk"(3.12.13 实测), + # 而这串字会经 server 变成用户看到的 SSE 进度。 + progress.step("derive", 0, 3, f"{action.action.value}: i2v 生成视频") + # 母版按动作预处理:jump 要在顶部补空间,否则角色腾空时头顶顶出视频画面被裁 + framed = prepare_master(master, action.action.value) + video = self._video.i2v(framed, self._build_prompt(action), seconds=5) + + dense = extract_all_frames_bytes(video) + # 跨动作一致性:用视频首帧(=母版姿态)的角色高当共同定标基准。各动作都从同一母版 + # 起手,故此值一致 —— 否则各动作按自己最高帧定标,切状态时角色会忽大忽小。 + ref_h = None + if dense: + _first = _img(self._matte.cutout(_png(dense[0]))) + _ys, _ = np.where(np.asarray(_first)[:, :, 3] > 128) + ref_h = float(_ys.max() - _ys.min()) if len(_ys) else None + if action.action in CYCLIC_ACTIONS: + progress.step("derive", 1, 3, f"步态周期取 {n} 帧(无缝 loop)+ 抠图") + picked = pick_cycle(dense, n) # 单周期闭环(#21) + else: + progress.step("derive", 1, 3, f"裁动作区间取 {n} 帧(不闭环)+ 抠图") + kind = "airborne" if action.action is ActionType.JUMP else "swing" + picked = pick_oneshot(dense, n, kind=kind) # 一次性动作:裁起止 + cut = [_img(self._matte.cutout(_png(im))) for im in picked] + + # 风格化按需(见 ActionSpec.stylize):none=保留 i2v 画风(插画/伪 3D 角色); + # pixel=像素化。原生像素角色**按母版规格**做:吸附母版像素网格 + 锁母版色板, + # 顺带消掉首帧 JPG / H.264 在硬边留下的灰颗粒(实测:通用降采样+量化反而更糊)。 + if action.stylize is Stylize.NONE: + progress.step("derive", 2, 3, "保留 i2v 画风(不像素化)") + return [_png(im) for im in cut] + + target_h, palette = action.pixel_h, None + try: + logical_h, pal = master_pixel_spec(_img(master)) # 用原始母版,不用补过边的 + if logical_h > 8: # 母版确为像素画 → 按它的规格走 + target_h, palette = logical_h, pal + except Exception: # 母版非像素画/量不出 → 回退通用量化 + pass + progress.step( + "derive", 2, 3, + f"像素化(h={target_h}{'·锁母版色板' if palette is not None else '·通用量化'})", + ) + pix = pixelate_frames( + cut, target_h=target_h, palette_size=action.palette_size, + palette=palette, ref_height=ref_h, + ) + return [_png(p) for p in pix] + + +class PerFrameStrategy(DerivationStrategy): + """离散姿势（hit 等，需单帧可编辑）：逐帧图生图 → 抠图。**未实现**（#53）。 + + 这条路线的价值在"单帧可重画"，与 i2v 是不同的产品能力，不能拿 i2v 顶替。 + """ + + route = GenRoute.PER_FRAME + + def __init__(self, image: ImageProvider, matte: MatteProvider) -> None: + self._image = image + self._matte = matte + + def derive( + self, + card: CharacterCard, + action: ActionSpec, + master: bytes, + progress: ProgressPort, + ) -> list[bytes]: + # 显式抛错，**不返回空帧**。曾经的桩实现 `return [b""] * n_frames` 会让调用方拿到 + # 一个"帧数对、时长对、无异常"的 GeneratedAction —— server 照常把 N 个 0 字节文件 + # 传上对象存储、写进 character_data，用户看到 N 张裂图，且排查时不会想到是路线没实现。 + # 未实现就要在边界上炸，不能让空数据流下去。 + raise NotImplementedError( + f"生成路线 {self.route.value} 尚未实现（动作 {action.action.value}）。" + "见 1024XEngineer/Windup#53。" + ) + + +# 注:曾有 ProcIdleStrategy(GenRoute.PROC_IDLE)—— 待机走"母版抠图 + 程序化局部躯干呼吸" +# 的零 API 路线(Idle-B,#53 原设计)。**2026-08-07 定案放弃**:程序化呼吸做不出可用效果, +# idle 统一走 i2v、认这份钱。GenRoute.PROC_IDLE 一并移除,不留没有实现的枚举值。 diff --git a/backend/packages/app/src/windup_app/bootstrap/app.py b/backend/packages/app/src/windup_app/bootstrap/app.py index fd997e30..b3d40466 100644 --- a/backend/packages/app/src/windup_app/bootstrap/app.py +++ b/backend/packages/app/src/windup_app/bootstrap/app.py @@ -21,6 +21,8 @@ from windup_app.server.workflow_run.model import WorkflowRun # noqa: F401 from windup_app.web.api.auth import router as auth_router from windup_app.web.api.character import router as character_router +from windup_app.server.orchestrator import task_repo +from windup_app.server.orchestrator.executor import run_action_task, run_image_task from windup_app.web.api.generation import router as generation_router from windup_app.web.api.media import router as media_router from windup_app.web.api.project import router as project_router @@ -90,6 +92,16 @@ def create_app() -> FastAPI: app.include_router(workflow_run_router) app.include_router(media_router) app.include_router(generation_router) + # 生成任务的后台执行器挂到 app.state:端点只建 PENDING 记录立即返回,真正的 + # 图生图/i2v 在后台线程跑。放在 state 而不是 import 到 web 层,是因为 + # import-linter 的分层契约禁止 app.web 直连 ai_engine,而 executor 要调它。 + app.state.run_action_task = run_action_task + app.state.run_image_task = run_image_task + + # task_repo 状态变更时自动推 SSE。延迟 import 避免与 generation 模块循环依赖。 + from windup_app.web.api.generation import event_bus + + task_repo.bind_event_bus(event_bus) register_exception_handlers(app) return app diff --git a/backend/packages/app/src/windup_app/server/orchestrator/__init__.py b/backend/packages/app/src/windup_app/server/orchestrator/__init__.py index c21a7b85..2cea0d63 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/__init__.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/__init__.py @@ -1,4 +1,9 @@ -"""生成任务领域。""" +"""生成任务编排(orchestrator):提交 / 调度 / 查询生成任务。 + +本包只做**任务编排调度**——建任务记录、后台驱动执行、查询状态;实际 AI 生成 +(调 ai_engine)在 :mod:`.executor` 后台跑。原名 ``generation``,更名为 ``orchestrator`` +以准确表达职责(调度而非生成本身)。 +""" from windup_app.server.orchestrator.model import ( ActionType, @@ -7,9 +12,12 @@ CharacterActionOutput, CharacterImageInput, GenerationTask, + GenerationTaskRecord, GenerationType, TaskStatus, ) +from windup_app.server.orchestrator.service import service as generation_service +from windup_app.server.orchestrator import task_repo __all__ = [ "ActionType", @@ -18,6 +26,9 @@ "CharacterActionOutput", "CharacterImageInput", "GenerationTask", + "GenerationTaskRecord", "GenerationType", "TaskStatus", + "generation_service", + "task_repo", ] diff --git a/backend/packages/app/src/windup_app/server/orchestrator/_fetch.py b/backend/packages/app/src/windup_app/server/orchestrator/_fetch.py new file mode 100644 index 00000000..0d8e1403 --- /dev/null +++ b/backend/packages/app/src/windup_app/server/orchestrator/_fetch.py @@ -0,0 +1,71 @@ +"""受限的服务端取图 —— 只允许拉自家对象存储，且限响应体大小。 + +为什么需要它（2026-08-10，机器审逮到）：``executor`` 原先直接 +``httpx.get(input.reference_image_urls[0])``，而那个 URL 来自已认证请求的请求体。 +服务端替调用方发起请求，等于把服务器当跳板： + +- ``http://127.0.0.1:8000/...`` 打到自己身上，绕过鉴权中间件访问内网端点； +- 云环境的实例元数据服务（各家都是一个固定的 link-local 地址）会吐出临时凭证； +- 私网地址段可以拿来探测内网拓扑； +- 重定向能把一个看起来合法的域名换成上面任意一种，所以**必须禁跟随重定向**； +- 响应体无上限时，一个指向巨大文件的 URL 就能把 worker 的内存吃光。 + +设计取向是**白名单**而不是黑名单：黑名单要穷举 127/8、10/8、172.16/12、192.168/16、 +169.254/16、::1、fc00::/7、以及各种十进制/八进制/IPv6-mapped 写法，漏一条就等于没做。 +而这里的业务只需要拉自家 bucket 的图（母版与参考图都是先经 ``/media/upload`` 传上去的）， +所以直接卡"必须是 ``storage_settings.download_base`` 前缀"。 + +代价：调用方不能再传外部图床链接。这是刻意的——真要支持，该走一个显式的"导入外部素材" +入口，在那里做完整的地址校验与配额，而不是让生成链路顺手具备任意 URL 抓取能力。 +""" +from __future__ import annotations + +import httpx + +from windup_framework.config.storage import settings as storage_settings + +__all__ = ["MAX_FETCH_BYTES", "FetchNotAllowed", "fetch_own_media"] + +# 单张图的上限。母版是 1024² 级的 PNG（实测 860~970 KB），16 MiB 留了足够余量， +# 又不至于让一个恶意 URL 拖垮 worker 内存。 +MAX_FETCH_BYTES = 16 * 1024 * 1024 + + +class FetchNotAllowed(ValueError): + """URL 不在允许范围内，或响应体超限。属调用方输入问题（4xx），不该重试。""" + + +def fetch_own_media(url: str, *, timeout: float = 30.0) -> bytes: + """取自家对象存储上的一张图。非自家地址、重定向、超大响应一律拒绝。""" + base = storage_settings.download_base + if not base: + raise FetchNotAllowed( + "对象存储下载域名未配置（WINDUP_STORAGE_BUCKET_DOMAIN），无法校验来源" + ) + if not url.startswith(f"{base}/"): + raise FetchNotAllowed( + f"只允许拉自家对象存储（{base}）上的素材，收到 {url[:80]!r}。" + "外部图片请先经 POST /media/upload 传入。" + ) + + # follow_redirects=False：跟随重定向会让白名单失效 —— 自家域名返回 302 指向 + # 元数据服务，校验就白做了。自家 bucket 直读不需要重定向。 + with httpx.Client(timeout=timeout, follow_redirects=False) as client: + with client.stream("GET", url) as resp: + resp.raise_for_status() + declared = resp.headers.get("content-length") + if declared and int(declared) > MAX_FETCH_BYTES: + raise FetchNotAllowed( + f"素材 {int(declared)} 字节，超过上限 {MAX_FETCH_BYTES}" + ) + # Content-Length 可以缺失或撒谎，故边读边计数。 + chunks: list[bytes] = [] + total = 0 + for chunk in resp.iter_bytes(): + total += len(chunk) + if total > MAX_FETCH_BYTES: + raise FetchNotAllowed( + f"素材超过上限 {MAX_FETCH_BYTES} 字节（已读 {total}）" + ) + chunks.append(chunk) + return b"".join(chunks) diff --git a/backend/packages/app/src/windup_app/server/orchestrator/executor.py b/backend/packages/app/src/windup_app/server/orchestrator/executor.py new file mode 100644 index 00000000..492e7c06 --- /dev/null +++ b/backend/packages/app/src/windup_app/server/orchestrator/executor.py @@ -0,0 +1,453 @@ +"""动作生成后台编排(调 ai_engine)。 + +编排链:``mark RUNNING → 取母版 → ai_engine 出帧 → 逐帧上传对象存储 → 写回结果/COMPLETED``。 +异常兜底为 FAILED,不抛。 + +**分层**:本模块调 ai_engine,故 web/worker **不得 import 本模块**(否则牵出 ai_engine, +违反"入口层不经 ai_engine 直连"门禁)。由 bootstrap(composition root)import + 注入 +``app.state``,web 端从 ``request.app.state`` 运行期取回调度,不产生静态依赖。 + +依赖(generator / upload / 取母版 / session 工厂)全可注入,缺省用真实实现(懒加载, +避免 import-time 触发 AI 配置)。测试注入桩即可离线跑通,不联网、不碰对象存储。 +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from sqlalchemy.orm import Session + +from windup_common.models import ActionSpec, ActionType as EngineActionType, CharacterCard + +from windup_app.server.orchestrator import task_repo +from windup_app.server.orchestrator._fetch import fetch_own_media +from windup_app.server.orchestrator.model import ( + CharacterActionInput, + CharacterImageInput, + TaskStatus, +) + +if TYPE_CHECKING: + from windup_ai_engine.ports import CharacterGeneratorPort, ProgressPort + +logger = logging.getLogger("windup.generation.executor") + +_ACTION_RESULT = "character_action" # task_repo._deserialize_result 按此标签反序列化 + +# ── 项目全局约束(Project 表)→ 统合喂给生成逻辑 ───────────────────────── +# character_perspective 游戏视角:1=横版(侧视) 2=俯视 3=2.5D → 生成朝向/视角 +_PERSPECTIVE_FACING: dict[int, str] = {1: "side", 2: "front", 3: "front"} +_PERSPECTIVE_VIEW: dict[int, str] = { + 1: "side view, horizontal side-scroller", + 2: "top-down view", + 3: "2.5D three-quarter view", +} +# directional_movement 移动方向:1=单向 2=四向 3=八向 → 需生成的方向数 +_MOVEMENT_DIRECTIONS: dict[int, int] = {1: 1, 2: 4, 3: 8} + + +@dataclass +class ProjectConstraints: + """从 Project 取的全局生成约束,统一约束角色图/动作生成。""" + + facing: str = "side" # character_perspective → 朝向(须与母版一致 #35) + view: str = "side view, horizontal side-scroller" + perspective: int = 1 # 1横版 2俯视 3 2.5D + directions: int = 1 # directional_movement → 方向数(1/4/8) + sprite_w: int = 256 # 输出/切帧尺寸(关键) + sprite_h: int = 256 + style: str = "" # game_style 画风 + stylize: str = "none" # 由 style 推:像素游戏 → pixel + sprite_sample_url: str = "" # 项目风格参考图 URL + + +def _load_constraints(session: Session, project_id: int | None) -> ProjectConstraints: + """查 Project 组装全局约束;无 project_id / 查不到 → 缺省。""" + if project_id is None: + return ProjectConstraints() + from windup_app.server.project.service import SqlAlchemyProjectService + + p = SqlAlchemyProjectService().get_project(session, project_id) + if p is None: + return ProjectConstraints() + style = p.game_style or "" + is_pixel = "pixel" in style.lower() or "像素" in style + return ProjectConstraints( + facing=_PERSPECTIVE_FACING.get(p.character_perspective, "side"), + view=_PERSPECTIVE_VIEW.get(p.character_perspective, _PERSPECTIVE_VIEW[1]), + perspective=p.character_perspective, + directions=_MOVEMENT_DIRECTIONS.get(p.directional_movement, 1), + sprite_w=p.sprite_width, + sprite_h=p.sprite_height, + style=style, + stylize="pixel" if is_pixel else "none", + sprite_sample_url=p.sprite_sample_url or "", + ) + + +def _fit_to(png: bytes, w: int, h: int, *, smooth: bool = False) -> bytes: + """把图等比缩放进 w×h(透明补边),落实尺寸约束。 + + ``smooth`` 决定重采样:序列帧是像素画,必须 NEAREST(插值会把硬边糊成灰边、 + 并引入调色板外的颜色);全彩角色母版反过来,NEAREST 缩图会明显锯齿,用 LANCZOS。 + """ + import io + + from PIL import Image + + im = Image.open(io.BytesIO(png)).convert("RGBA") + if im.size == (w, h): + return png + fitted = im.copy() + fitted.thumbnail((w, h), Image.LANCZOS if smooth else Image.NEAREST) + canvas = Image.new("RGBA", (w, h), (0, 0, 0, 0)) + canvas.alpha_composite(fitted, ((w - fitted.width) // 2, (h - fitted.height) // 2)) + buf = io.BytesIO() + canvas.save(buf, "PNG") + return buf.getvalue() + + +def _require_size(png: bytes, w: int, h: int) -> bytes: + """核对引擎交付帧确实是项目要的尺寸,不对就报错 —— **不做静默补救**。 + + 这里以前是 ``_fit_to``:尺寸对不上就缩放补边。看着稳,实际是把"引擎没按尺寸出帧" + 这件事悄悄抹平,代价是脚线对齐被破坏(见 ``_produce_action`` 的说明)。尺寸现在由 + 引擎按 ``canvas`` 负责,对不上说明生成侧出了问题,该让它响,而不是交付一批对齐 + 坏掉的帧 —— 那正是本仓最忌讳的"看起来成功的错产物"。 + """ + import io + + from PIL import Image + + size = Image.open(io.BytesIO(png)).size + if size != (w, h): + raise ValueError( + f"引擎交付帧尺寸 {size[0]}×{size[1]} 与项目约束 {w}×{h} 不一致;" + "生成侧未按 canvas 出帧,不做静默缩放补救。" + ) + return png + + +class _LogProgress: + """进度上报占位:MVP 无 SSE,记日志即可。""" + + def step(self, stage: str, i: int, total: int, note: str = "") -> None: + logger.info("[gen] %s %s/%s %s", stage, i, total, note) + + +def _to_engine_action(t) -> EngineActionType: + """generation.ActionType → 引擎 common.ActionType(按值映射)。 + + walk/idle/attack 直通;custom 等引擎未覆盖的类型暂不支持视频路线。 + """ + try: + return EngineActionType(t.value) + except ValueError as e: + raise ValueError(f"动作类型 {t.value!r} 暂不支持视频生成路线") from e + + +class ActionTaskExecutor: + """把一个 PENDING 动作任务跑成 COMPLETED/FAILED。""" + + def __init__( + self, + *, + generator: CharacterGeneratorPort | None = None, + upload: Callable[[bytes], str] | None = None, + fetch_master: Callable[[CharacterActionInput], bytes] | None = None, + fetch_constraints: Callable[[Session, int | None], ProjectConstraints] | None = None, + session_factory: Callable[[], Session] | None = None, + ) -> None: + self._generator = generator # None → 懒加载真实装配 + self._upload = upload # None → 真实对象存储上传 + self._fetch_master = fetch_master # None → 下载 reference_image_urls[0] + self._fetch_constraints = fetch_constraints # None → 查 project 全局约束 + self._session_factory = session_factory # None → SessionLocal + + def run_action_task( + self, + task_id: int, + input: CharacterActionInput, + project_id: int | None = None, + *, + session: Session | None = None, + ) -> None: + """跑一个动作任务;异常兜底为 FAILED,不抛。 + + 先从 ``project`` 取全局约束(朝向/画风/尺寸/方向)再调 ai_engine。``session`` + 缺省时自开一个(后台场景);测试可传入自己的 session。 + """ + own = session is None + session = session or self._make_session() + try: + task_repo.update_status(session, task_id, TaskStatus.RUNNING) + if own: + session.commit() + + cons = (self._fetch_constraints or _load_constraints)(session, project_id) + result = self._produce_action(input, cons) + task_repo.update_result(session, task_id, _ACTION_RESULT, result) + if own: + session.commit() + except Exception as exc: # noqa: BLE001 —— 兜底任何生成/上传/网络异常 + logger.exception("动作任务 %s 失败", task_id) + task_repo.update_status( + session, task_id, TaskStatus.FAILED, error_message=str(exc), + ) + if own: + session.commit() + finally: + if own: + session.close() + + # -- 内部 -------------------------------------------------------------- + + def _produce_action(self, input: CharacterActionInput, cons: ProjectConstraints) -> dict: + """母版 → ai_engine 按项目尺寸出帧 → 逐帧上传 → 组结果 dict。 + + 项目约束落实:``facing`` 随视角、``stylize`` 随画风(像素游戏→像素化)、 + 输出帧尺寸随 ``sprite_w×sprite_h``。方向数(directions)MVP 先出主方向, + 四向/八向为扩展(需多次生成或镜像)。 + + **尺寸是传给引擎的,不是拿到帧再缩的。** 这里曾对每帧再做一次 + ``_fit_to(png, sprite_w, sprite_h)``:引擎恒出 256,项目要 512 就等于二次 + 重采样。而 ``_fit_to`` 用 ``Image.thumbnail`` —— 它**只缩不放**,放大方向 + 根本不放大,只是把 256 的帧原尺寸居中贴进 512 画布,于是引擎刚对齐好的脚线 + 0.92 被挪到 0.709(2026-08-11 实测),角色不站在地上、跨动作对齐一并失效。 + 现在把 ``canvas`` 交给引擎,它一次就出到项目尺寸,那一步整个不存在了。 + """ + if cons.directions > 1: + logger.info("项目要求 %s 方向,MVP 先出主方向(多方向待扩展)", cons.directions) + master = (self._fetch_master or self._download_master)(input) + # 视频 i2v 没有独立的 style reference 字段,风格约束走提示词文字 + desc_parts = [input.custom_prompt or ""] + if cons.style: + desc_parts.append(f"Art style: {cons.style}") + card = CharacterCard(name=f"char-{input.character_id}", desc=" ".join(desc_parts)) + action = ActionSpec( + action=_to_engine_action(input.action_type), + poses=[""] * input.num_frames, + facing=cons.facing, + stylize=cons.stylize, + ) + progress: ProgressPort = _LogProgress() + generated = self._get_generator().generate( + card, action, master, progress, canvas=(cons.sprite_w, cons.sprite_h) + ) + + upload = self._upload or self._upload_frame + frames = [ + {"index": i, + "image_url": upload(_require_size(png, cons.sprite_w, cons.sprite_h)), + "duration_ms": dur} + for i, (png, dur) in enumerate(zip(generated.frames, generated.durations)) + ] + return {"type": "character_action", "action_type": input.action_type.value, "frames": frames} + + def _get_generator(self) -> CharacterGeneratorPort: + """懒装配真实 CharacterGenerator(视频路线 + 桩路线)。""" + if self._generator is None: + from windup_ai_engine.impl import CharacterGenerator + from windup_ai_engine.strategy.concrete import ( + PerFrameStrategy, + VideoFrameStrategy, + ) + from windup_common.models import GenRoute + from windup_framework.providers import ( + OnnxU2NetMatteProvider, + SufyImageProvider, + SufyVideoProvider, + ) + + matte = OnnxU2NetMatteProvider() + video = SufyVideoProvider() + image = SufyImageProvider() + # 只装当前 GenRoute 真有的路线。曾多装一个 PROC_IDLE:该枚举值与 + # ProcIdleStrategy 都已随"程序化待机放弃"一起删除,而这行留着,于是**每个** + # 动作任务都在 import 期 AttributeError —— 注入 generator 的测试走不到这条 + # 装配路径,所以测试全绿而真实调用全崩(FennoAI 逮到,2026-08-10)。 + # 加一条断言:将来 GenRoute 新增成员时,漏装会在这里立刻暴露,而不是等到 + # 某个动作第一次被请求。 + strategies = { + GenRoute.VIDEO_I2V: VideoFrameStrategy(video, matte), + GenRoute.PER_FRAME: PerFrameStrategy(image, matte), + } + missing = set(GenRoute) - set(strategies) + if missing: + raise RuntimeError( + f"GenRoute 新增了 {sorted(r.value for r in missing)} 但 executor 未装配;" + "补上或在此显式说明为何不装。" + ) + self._generator = CharacterGenerator(strategies) + return self._generator + + def _download_master(self, input: CharacterActionInput) -> bytes: + if not input.reference_image_urls: + raise ValueError("缺少母版:reference_image_urls 为空") + # 只允许拉自家对象存储:这个 URL 来自请求体,直接 httpx.get 等于把服务器 + # 当跳板(可打 loopback / 云元数据服务 / 私网)。详见 _fetch 模块 docstring。 + return fetch_own_media(input.reference_image_urls[0]) + + def _upload_frame(self, png: bytes) -> str: + from windup_app.server.media.model import MediaCategory, MediaUploadInput + from windup_app.server.media.service import service as media_service + + meta = MediaUploadInput( + filename="frame.png", + content_type="image/png", + size=len(png), + category=MediaCategory.ACTION_FRAME, + ) + return media_service.upload(png, meta).url + + def _make_session(self) -> Session: + if self._session_factory is not None: + return self._session_factory() + from windup_framework.db.session import SessionLocal + + return SessionLocal() + + +_IMAGE_RESULT = "character_image" # task_repo._deserialize_result 按此标签反序列化 + + +class ImageTaskExecutor: + """跑角色图片生成任务:参考图 + prompt → 图生图 → 上传 → 回写 image_url。""" + + def __init__( + self, + *, + image=None, # None → 懒加载 SufyImageProvider + upload: Callable[[bytes], str] | None = None, # None → 真实对象存储上传 + fetch_ref: Callable[[str], bytes] | None = None, # None → 下载 reference_image_url + session_factory: Callable[[], Session] | None = None, + ) -> None: + self._image = image + self._upload = upload + self._fetch_ref = fetch_ref + self._session_factory = session_factory + + def run_image_task( + self, + task_id: int, + input: CharacterImageInput, + project_id: int | None = None, + *, + session: Session | None = None, + ) -> None: + own = session is None + session = session or self._make_session() + try: + task_repo.update_status(session, task_id, TaskStatus.RUNNING) + if own: + session.commit() + cons = _load_constraints(session, project_id) # 角色图也受项目约束 + urls = self._produce_image(input, cons) + task_repo.update_result(session, task_id, _IMAGE_RESULT, { + "type": "character_image", + "image_urls": urls, + }) + if own: + session.commit() + except Exception as exc: # noqa: BLE001 —— 兜底 + logger.exception("图片任务 %s 失败", task_id) + task_repo.update_status(session, task_id, TaskStatus.FAILED, error_message=str(exc)) + if own: + session.commit() + finally: + if own: + session.close() + + def _produce_image(self, input: CharacterImageInput, cons: ProjectConstraints) -> list[str]: + """根据项目约束决定生成模式,返回 URL 列表。 + + 模式判断: + - 项目有 sprite_sample_url → **图生图**: 风格参考图 + 提示词 + - 项目无 sprite_sample_url → **文生图**: 纯提示词 + 用户传入的 reference_image_url 始终作为角色一致性参考(可选)。 + """ + fetch = self._fetch_ref or self._download + refs: list[bytes] = [] + has_style_ref = False + + # 1. 角色参考图(用户传入,可选,做角色一致性约束) + char_url = (input.reference_image_url or "").strip() + if char_url and char_url.lower() not in ("null", "none", ""): + refs.append(fetch(char_url)) + + # 2. 风格参考图(项目级,有 sprite_sample_url 时走图生图模式) + style_url = (cons.sprite_sample_url or "").strip() + if style_url and style_url.lower() not in ("null", "none", ""): + try: + refs.append(fetch(style_url)) + has_style_ref = True + except Exception: + pass # 风格参考图下载失败不阻断 + + # 3. 构建提示词 + base = input.prompt or "Clean full-body character reference of the figure in the image." + parts = [base, f"{cons.view}, full body head to feet, centered."] + if cons.style: + parts.append(f"Art style: {cons.style}.") + parts.append("Plain light-gray background, no shadow.") + + # 图生图模式:明确标注两张图的各自用途 + if has_style_ref: + prefix = ( + "This is an image-to-image task. " + "The first image is the CHARACTER reference — preserve its identity. " + "The second image is the STYLE reference — follow its art style, " + "color palette, and rendering technique. " + ) + parts.insert(0, prefix) + + prompt = " ".join(parts) + + image_gen = self._get_image() + upload = self._upload or self._upload_image + urls: list[str] = [] + for _ in range(max(1, input.num_images)): + img = image_gen.gen_image(prompt, refs) + # 请求里的 width/height 此前被丢掉:入口收下并校验过它们(_validate_project_size), + # 而 ImageProvider.gen_image 没有尺寸参数,模型出多大就返多大 —— 又一个"接了不 + # 履约"的字段(2026-08-10 对抗复查发现)。模型本身不吃宽高,所以在这里落实。 + urls.append(upload(_fit_to(img, input.width, input.height, smooth=True))) + return urls + + def _get_image(self): + if self._image is None: + from windup_framework.providers import SufyImageProvider + + self._image = SufyImageProvider() + return self._image + + def _download(self, url: str) -> bytes: + # 同 _download_master:参考图 URL 由调用方给,必须走白名单取图。 + return fetch_own_media(url) + + def _upload_image(self, png: bytes) -> str: + from windup_app.server.media.model import MediaCategory, MediaUploadInput + from windup_app.server.media.service import service as media_service + + meta = MediaUploadInput( + filename="character.png", content_type="image/png", + size=len(png), category=MediaCategory.REFERENCE_IMAGE, + ) + return media_service.upload(png, meta).url + + def _make_session(self) -> Session: + if self._session_factory is not None: + return self._session_factory() + from windup_framework.db.session import SessionLocal + + return SessionLocal() + + +# 默认执行器(真实依赖);bootstrap 取 run_action_task / run_image_task 注入 app.state +executor = ActionTaskExecutor() +run_action_task = executor.run_action_task +image_executor = ImageTaskExecutor() +run_image_task = image_executor.run_image_task diff --git a/backend/packages/app/src/windup_app/server/orchestrator/interface.py b/backend/packages/app/src/windup_app/server/orchestrator/interface.py index 83e38aa5..4d0a1d57 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/interface.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/interface.py @@ -22,6 +22,8 @@ from abc import ABC, abstractmethod +from sqlalchemy.orm import Session + from windup_app.server.orchestrator.model import ( CharacterActionInput, CharacterImageInput, @@ -35,7 +37,10 @@ class GenerationService(ABC): # -- 任务提交 ------------------------------------------------------------ @abstractmethod - def generate_character_image(self, input: CharacterImageInput) -> GenerationTask: + def generate_character_image( + self, session: Session, *, user_id: int, + project_id: int | None, input: CharacterImageInput, + ) -> GenerationTask: """提交角色图片生成任务。 入参包含参考图 URL 和 prompt 等参数；出参为 ``CharacterImageOutput``， @@ -43,7 +48,10 @@ def generate_character_image(self, input: CharacterImageInput) -> GenerationTask """ @abstractmethod - def generate_character_action(self, input: CharacterActionInput) -> GenerationTask: + def generate_character_action( + self, session: Session, *, user_id: int, + project_id: int | None, input: CharacterActionInput, + ) -> GenerationTask: """提交角色动作生成任务。 入参包含角色 ID、动作类型和参考素材；出参为 ``CharacterActionOutput``， @@ -53,7 +61,9 @@ def generate_character_action(self, input: CharacterActionInput) -> GenerationTa # -- 查询 ---------------------------------------------------------------- @abstractmethod - def get_task(self, project_id: int, task_id: int) -> GenerationTask | None: + def get_task( + self, session: Session, project_id: int, task_id: int, + ) -> GenerationTask | None: """查询任务状态与结果。 返回完整的 ``GenerationTask``，前端根据 ``status`` 判断是否完成， diff --git a/backend/packages/app/src/windup_app/server/orchestrator/model.py b/backend/packages/app/src/windup_app/server/orchestrator/model.py index 36fdab58..2558f416 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/model.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/model.py @@ -9,6 +9,12 @@ from datetime import datetime, timezone from enum import StrEnum +from sqlalchemy import BigInteger, DateTime, Integer, JSON, Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from windup_framework.db import Base + # -- 枚举 ---------------------------------------------------------------- @@ -25,6 +31,7 @@ class ActionType(StrEnum): WALK = "walk" IDLE = "idle" + JUMP = "jump" ATTACK = "attack" CUSTOM = "custom" @@ -124,3 +131,55 @@ class GenerationTask: @property def is_terminal(self) -> bool: return self.status in (TaskStatus.COMPLETED, TaskStatus.FAILED) + + +# -- ORM ----------------------------------------------------------------- + + +class GenerationTaskRecord(Base): + """生成任务持久化记录。 + + ``input_payload`` 和 ``result`` 以 JSON 存储；``result_type`` 标识 + ``result`` 的具体类型，读出后按类型反序列化为对应 dataclass。 + """ + + __tablename__ = "windup_generation_task" + + id: Mapped[int] = mapped_column( + BigInteger().with_variant(Integer, "sqlite"), + primary_key=True, + autoincrement=True, + ) + user_id: Mapped[int] = mapped_column(BigInteger, nullable=False) + project_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + task_type: Mapped[str] = mapped_column( + Text, nullable=False, + default=GenerationType.CHARACTER_IMAGE.value, + ) + status: Mapped[str] = mapped_column( + Text, nullable=False, + default=TaskStatus.PENDING.value, + ) + input_payload: Mapped[dict] = mapped_column( + JSON().with_variant(JSONB, "postgresql"), + nullable=False, + default=dict, + ) + result_type: Mapped[str | None] = mapped_column(Text, nullable=True) + result: Mapped[dict | None] = mapped_column( + JSON().with_variant(JSONB, "postgresql"), + nullable=True, + ) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + + create_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + update_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + ) diff --git a/backend/packages/app/src/windup_app/server/orchestrator/service.py b/backend/packages/app/src/windup_app/server/orchestrator/service.py new file mode 100644 index 00000000..ab46abac --- /dev/null +++ b/backend/packages/app/src/windup_app/server/orchestrator/service.py @@ -0,0 +1,56 @@ +"""生成任务领域服务(提交 + 查询)。 + +:class:`AiGenerationService` 只负责**建任务记录 + 查任务**——web 层依赖本模块。 +实际 AI 生成(调 ai_engine)在 :mod:`.executor` 后台跑,本模块**不碰 ai_engine**, +以满足"入口层(web/worker)不经 ai_engine 直连"的分层门禁(web → service 不得牵出 ai_engine)。 + +无状态:``session`` 由调用方按请求传入,本对象作模块级单例(:data:`service`)。 +""" + +from __future__ import annotations + +import dataclasses + +from sqlalchemy.orm import Session + +from windup_app.server.orchestrator import task_repo +from windup_app.server.orchestrator.interface import GenerationService +from windup_app.server.orchestrator.model import ( + CharacterActionInput, + CharacterImageInput, + GenerationTask, + GenerationType, +) + + +class AiGenerationService(GenerationService): + """生成任务服务:提交(建 PENDING 记录)+ 查询。生成执行在 executor 后台。""" + + def generate_character_image( + self, session: Session, *, user_id: int, project_id: int | None = None, + input: CharacterImageInput, + ) -> GenerationTask: + return task_repo.create_task( + session, user_id=user_id, project_id=project_id, + task_type=GenerationType.CHARACTER_IMAGE, + input_payload=dataclasses.asdict(input), + ) + + def generate_character_action( + self, session: Session, *, user_id: int, project_id: int | None = None, + input: CharacterActionInput, + ) -> GenerationTask: + """建动作生成任务(PENDING)并返回;实际生成由 executor 后台跑,前端轮询 get_task。""" + return task_repo.create_task( + session, user_id=user_id, project_id=project_id, + task_type=GenerationType.CHARACTER_ACTION, + input_payload=dataclasses.asdict(input), + ) + + def get_task( + self, session: Session, project_id: int, task_id: int, + ) -> GenerationTask | None: + return task_repo.get_task(session, task_id) + + +service = AiGenerationService() diff --git a/backend/packages/app/src/windup_app/server/orchestrator/task_repo.py b/backend/packages/app/src/windup_app/server/orchestrator/task_repo.py new file mode 100644 index 00000000..d2b60b5c --- /dev/null +++ b/backend/packages/app/src/windup_app/server/orchestrator/task_repo.py @@ -0,0 +1,231 @@ +"""生成任务数据访问层。 + +纯 CRUD 操作，不含业务逻辑。所有函数接收 ``session: Session``， +由调用方（FastAPI ``get_session`` 依赖）管理事务边界——本模块只 +``flush`` 不 ``commit``。 + +状态变更时自动向 EventBus 推送完整 task 数据（若已绑定）， +供 SSE 端点实时推送给前端，替代轮询。 +""" + +from __future__ import annotations + +import dataclasses +import logging +from datetime import datetime, timezone + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from windup_app.server.orchestrator.model import ( + CharacterActionOutput, + CharacterImageOutput, + GenerationTask, + GenerationTaskRecord, + GenerationType, + TaskStatus, +) + +logger = logging.getLogger("windup.task_repo") + +# EventBus 引用（bootstrap 中绑定，避免循环导入） +_event_bus = None + + +def bind_event_bus(event_bus) -> None: + """绑定 EventBus 实例（bootstrap 中调用）。""" + global _event_bus + _event_bus = event_bus + + +# 状态 → SSE 事件名。终态必须用独立事件名:web 层的 stream 靠事件名判断何时收尾 +# (``_TERMINAL_EVENTS``),一律发 "task_update" 的话那个判断永不成立 —— 客户端收到 +# completed 之后连接仍开着,只能靠心跳挂到超时,而端点带 retry: 3000,浏览器原生 +# EventSource 会每 3 秒重连、每次重收同一条 completed(2026-08-10 机器审逮到)。 +_STATUS_EVENT = { + TaskStatus.COMPLETED.value: "completed", + TaskStatus.FAILED.value: "failed", +} + + +def task_event_payload(task: GenerationTask) -> dict: + """SSE 事件体。**只有这一份实现**。 + + 抽成公开函数是因为有第二个发送点:SSE 端点在订阅时若发现任务已是终态,要立即补发 + 一条终态事件。那里再抄一份字段列表就是第二个真相源 —— 加字段时漏掉一处,客户端 + 会拿到形状不一致的两种同名事件。 + """ + return { + "id": task.id, + "user_id": task.user_id, + "project_id": task.project_id, + "task_type": task.task_type.value, + "status": task.status.value, + "input_payload": task.input_payload, + "result": dataclasses.asdict(task.result) if task.result else None, + "error_message": task.error_message, + } + + +def terminal_event_for(task: GenerationTask) -> str | None: + """任务已处于终态时对应的事件名;非终态返回 None。""" + return _STATUS_EVENT.get(task.status.value) + + +def _publish_task_update(task_id: int, task: GenerationTask) -> None: + """将完整 task 推送到 EventBus（若有订阅者）。 + + EventBus 的键是 ``(project_id, task_id)``(主线 #110:同一 task_id 在不同项目下互不 + 串流)。所以 ``task.project_id`` 为空时**发不到任何订阅者** —— 订阅方拿的键一定带 + 着一个真实的 project_id。这种情况记 warning 而不是静默 publish 到一个没人听的键上: + 静默发出去的话,现象是"任务确实在跑、状态也在落库,但前端进度条一动不动", + 而日志里一行异常都没有。 + """ + if _event_bus is None: + return + if task.project_id is None: + logger.warning( + "任务 %d 没有 project_id,SSE 事件无法投递(EventBus 按 (project_id, task_id) 索引)", + task_id, + ) + return + event = _STATUS_EVENT.get(task.status.value, "task_update") + _event_bus.publish(task.project_id, task_id, event, task_event_payload(task)) + + +# ── 写入 ───────────────────────────────────────────────────────────────── + + +def create_task( + session: Session, + *, + user_id: int, + project_id: int | None, + task_type: GenerationType, + input_payload: dict, +) -> GenerationTask: + """创建生成任务记录，返回领域对象。""" + record = GenerationTaskRecord( + user_id=user_id, + project_id=project_id, + task_type=task_type.value, + status=TaskStatus.PENDING.value, + input_payload=input_payload, + ) + session.add(record) + session.flush() + return _record_to_domain(record) + + +def update_status( + session: Session, + task_id: int, + status: TaskStatus, + *, + error_message: str | None = None, +) -> None: + """更新任务状态（可选附带错误信息）。""" + record = session.get(GenerationTaskRecord, task_id) + if record is None: + return + record.status = status.value + record.error_message = error_message + record.update_at = datetime.now(timezone.utc) + session.flush() + _publish_task_update(task_id, _record_to_domain(record)) + + +def update_result( + session: Session, + task_id: int, + result_type: str, + result: dict, +) -> None: + """写入任务结果。""" + record = session.get(GenerationTaskRecord, task_id) + if record is None: + return + record.result_type = result_type + record.result = result + record.status = TaskStatus.COMPLETED.value + record.update_at = datetime.now(timezone.utc) + session.flush() + _publish_task_update(task_id, _record_to_domain(record)) + + +# ── 读取 ───────────────────────────────────────────────────────────────── + + +def get_task(session: Session, task_id: int) -> GenerationTask | None: + """按 task_id 查询任务。""" + record = session.get(GenerationTaskRecord, task_id) + if record is None: + return None + return _record_to_domain(record) + + +def get_task_by_user( + session: Session, + user_id: int, + task_id: int, +) -> GenerationTask | None: + """按 user_id + task_id 查询（校验归属）。""" + stmt = select(GenerationTaskRecord).where( + GenerationTaskRecord.id == task_id, + GenerationTaskRecord.user_id == user_id, + ) + record = session.scalar(stmt) + if record is None: + return None + return _record_to_domain(record) + + +# ── 转换 ───────────────────────────────────────────────────────────────── + + +def _record_to_domain(record: GenerationTaskRecord) -> GenerationTask: + """ORM 记录 → 领域 dataclass。""" + result = _deserialize_result(record.result_type, record.result) + return GenerationTask( + id=record.id, + user_id=record.user_id, + project_id=record.project_id, + task_type=GenerationType(record.task_type), + status=TaskStatus(record.status), + input_payload=record.input_payload, + result=result, + error_message=record.error_message, + create_at=record.create_at, + update_at=record.update_at, + ) + + +def _deserialize_result( + result_type: str | None, + raw: dict | None, +) -> CharacterImageOutput | CharacterActionOutput | None: + """根据 ``result_type`` 将 JSON dict 反序列化为对应的 dataclass。""" + if raw is None or result_type is None: + return None + if result_type == "character_image": + return CharacterImageOutput( + type=raw.get("type", "character_image"), + image_urls=raw.get("image_urls", []), + ) + if result_type == "character_action": + from windup_app.server.orchestrator.model import CharacterActionFrame + + frames = [ + CharacterActionFrame( + index=f["index"], + image_url=f["image_url"], + duration_ms=f.get("duration_ms"), + ) + for f in raw.get("frames", []) + ] + return CharacterActionOutput( + type=raw.get("type", "character_action"), + action_type=raw.get("action_type", ""), + frames=frames, + ) + return None diff --git a/backend/packages/app/src/windup_app/web/api/generation.py b/backend/packages/app/src/windup_app/web/api/generation.py index b03e8afe..5b4e83df 100644 --- a/backend/packages/app/src/windup_app/web/api/generation.py +++ b/backend/packages/app/src/windup_app/web/api/generation.py @@ -30,6 +30,7 @@ from windup_framework.db import get_session from windup_app.server.character.model import Character +from windup_app.server.orchestrator import task_repo from windup_app.server.orchestrator.model import ( ActionType, GenerationTask, @@ -53,14 +54,27 @@ class _EventBus: - """任务进度内存发布-订阅。""" + """任务进度内存发布-订阅。 + + **publish 会被后台线程调用**(executor 在 daemon thread 里跑,经 task_repo 触发), + 而队列属于处理 SSE 请求的那个 event loop。``asyncio.Queue`` 不是线程安全的: + 跨线程 ``put_nowait`` 能把元素放进去,但唤醒 waiter 用的是 loop 内部调度, + 从别的线程调不会唤醒 —— 订阅者可能一直挂在 ``get()`` 上,直到下一次同 loop 内的 + 操作偶然把它带起来。故订阅时记下所属 loop,发布时经 ``call_soon_threadsafe`` + 回到那个 loop 上再入队(2026-08-10 机器审逮到)。 + """ def __init__(self) -> None: - self._queues: dict[tuple[int, int], list[asyncio.Queue]] = defaultdict(list) + # 键是 (project_id, task_id):同一个 task_id 在不同项目下互不串流(主线 #110)。 + # 值是 (queue, 它所属的 loop):不同订阅者可能来自不同 loop(多 worker / 测试里的 + # 临时 loop),不能只存一个全局 loop —— 见 publish 里的 call_soon_threadsafe。 + self._queues: dict[ + tuple[int, int], list[tuple[asyncio.Queue, asyncio.AbstractEventLoop]] + ] = defaultdict(list) async def subscribe(self, project_id: int, task_id: int) -> asyncio.Queue: queue: asyncio.Queue = asyncio.Queue() - self._queues[(project_id, task_id)].append(queue) + self._queues[(project_id, task_id)].append((queue, asyncio.get_running_loop())) return queue async def unsubscribe( @@ -71,10 +85,11 @@ async def unsubscribe( ) -> None: key = (project_id, task_id) subs = self._queues.get(key) - if subs and queue in subs: - subs.remove(queue) - if not subs: - del self._queues[key] + if not subs: + return + self._queues[key] = [(q, lp) for q, lp in subs if q is not queue] + if not self._queues[key]: + del self._queues[key] def publish( self, @@ -83,8 +98,32 @@ def publish( event: str, data: dict, ) -> None: - for queue in self._queues.get((project_id, task_id), []): - queue.put_nowait((event, data)) + """跨线程安全地投递。 + + executor 在 daemon thread 里跑,而队列属于处理 SSE 请求的那个 event loop。 + ``asyncio.Queue`` 不是线程安全的:跨线程 ``put_nowait`` 能把元素放进去,但唤醒 + waiter 用的是 loop 内部调度,从别的线程调不会唤醒 —— 订阅者可能一直挂在 + ``get()`` 上,直到下一次同 loop 内的操作偶然把它带起来。故订阅时记下所属 loop, + 发布时经 ``call_soon_threadsafe`` 回到那个 loop 上再入队。 + """ + try: + here = asyncio.get_running_loop() + except RuntimeError: + here = None # 从没有 loop 的线程调(executor daemon thread) + + for queue, loop in list(self._queues.get((project_id, task_id), [])): + if loop is here: + # 同一个 loop 内:直接入队。**不能一律走 call_soon_threadsafe** —— 那是 + # 异步调度,要等 loop 下一次迭代才真入队,于是"publish 完立刻 get_nowait" + # 会拿到空队列(主线 #110 的隔离用例正是这么写的)。 + queue.put_nowait((event, data)) + continue + try: + loop.call_soon_threadsafe(queue.put_nowait, (event, data)) + except RuntimeError: + # loop 已关闭(客户端断连后请求 loop 结束)。丢弃即可 —— 没有订阅者在等 + # 这条消息,而任务状态本身已落库,重连后靠 GET /tasks/{id} 取。 + logger.debug("SSE loop 已关闭,丢弃事件 task_id=%d event=%s", task_id, event) # 全局实例，挂到 app.state.event_bus @@ -99,25 +138,37 @@ def publish( class CharacterImageGenerateRequest(BaseModel): """提交角色图片生成任务。""" + # project_id 必填,它是归属校验的依据(见 _get_project_or_raise)。 + # 注:曾有 `user_id: int = Field(gt=0)`。归属者从 request.state.current_user 取, + # 请求体里那个字段既不被读、又让调用方以为自己能指定归属者 —— 填别人的 id 不报错 + # 也不生效,正是本仓最忌讳的"看起来生效的错"。已删。 project_id: int = Field(gt=0) reference_image_url: str | None = None prompt: str = "" negative_prompt: str = "" - width: int = 1024 - height: int = 1024 - num_images: int = 1 + # 三个上界都直通付费调用,必须在契约层卡住:num_images 是 provider 调用次数的 + # 循环上界,一个已认证请求填个大数就能绕过按请求计的限流、把成本拉到无上限 + # (2026-08-10 机器审逮到)。宽高上界按当前 i2v 与像素化管线的实际处理范围取。 + width: int = Field(default=1024, ge=64, le=2048) + height: int = Field(default=1024, ge=64, le=2048) + num_images: int = Field(default=1, ge=1, le=4) class CharacterActionGenerateRequest(BaseModel): """提交角色动作生成任务。""" + # project_id 必填,它是归属校验的依据(见 _get_project_or_raise)。 + # 注:曾有 `user_id: int = Field(gt=0)`。归属者从 request.state.current_user 取, + # 请求体里那个字段既不被读、又让调用方以为自己能指定归属者 —— 填别人的 id 不报错 + # 也不生效,正是本仓最忌讳的"看起来生效的错"。已删。 project_id: int = Field(gt=0) character_id: int = Field(gt=0) action_type: ActionType custom_prompt: str | None = None reference_video_url: str | None = None reference_image_urls: list[str] = Field(default_factory=list) - num_frames: int = 16 + # 同上:帧数决定抽帧与逐帧抠图的工作量,上界 64 已远超引擎能出的有效周期长度。 + num_frames: int = Field(default=16, ge=1, le=64) class GenerationTaskOut(BaseModel): @@ -245,15 +296,33 @@ async def stream_task( - ``failed``: 任务失败,携带错误信息 若客户端订阅时任务已处于终态,立即推送终态事件并关闭连接。 + + 归属是**两道**(2026-08-11 补齐,此前是一行 TODO):项目要属于当前用户 + (``_get_project_or_raise``),任务还要属于那个项目。只查项目不够 —— 任意已认证用户 + 拿自己的 project_id 配上别人的 task_id 就能订阅到别人的流,而事件体里带 result, + 即最终帧的对象存储 URL。两道都必须在 ``subscribe`` **之前**:放之后的话越权请求仍会 + 在 EventBus 上挂一个订阅者(照样收事件、只是响应体被丢弃),订阅表还会因为没人 + unsubscribe 而增长。 """ user_id = request.state.current_user.id _get_project_or_raise(session, project_id, user_id) - # TODO: 检查任务属于 project_id 及初始状态,若已终态立即推送 + task = task_repo.get_task(session, task_id) + if task is None or task.project_id != project_id: + raise BizException("任务不存在", code=BizCode.NOT_FOUND) + + # 终态快照要在订阅前读,订阅要紧跟其后 —— 两者之间若任务刚好终结,事件会丢。 + # 反过来(先订阅后读)则会重复发一次终态,客户端拿到两条 completed。 + terminal_event = task_repo.terminal_event_for(task) + queue = await event_bus.subscribe(project_id, task_id) - logger.debug("SSE 订阅: task_id=%d", task_id) + logger.debug("SSE 订阅: task_id=%d project_id=%d", task_id, project_id) async def _event_generator(): try: + if terminal_event is not None: + payload = json.dumps(task_repo.task_event_payload(task), ensure_ascii=False) + yield f"event: {terminal_event}\ndata: {payload}\n\n" + return while True: if await request.is_disconnected(): logger.debug("SSE 客户端断开: task_id=%d", task_id) diff --git a/backend/packages/framework/src/windup_framework/config/provider.py b/backend/packages/framework/src/windup_framework/config/provider.py index 57182ce5..faeede24 100644 --- a/backend/packages/framework/src/windup_framework/config/provider.py +++ b/backend/packages/framework/src/windup_framework/config/provider.py @@ -16,11 +16,24 @@ class AIProviderSettings(BaseSettings): provider: str = "openai-compatible" base_url: str = "https://api.openai.com/v1" api_key: str = "" - model: str = "" + model: str = "" # 通用兜底(chat 类调用),下面三个各自专用 timeout: float = 120.0 max_retries: int = 2 chat_completions_path: str = "/chat/completions" + # ── 各能力用哪个模型 ────────────────────────────────────────────────── + # 分成三个字段而不是共用上面那个 ``model``:三条能力同时在用不同模型,共用一个 + # 字段意味着换其中一个就把另外两个也换了。默认值即当前实测在用的型号, + # 部署侧可用 AI_VIDEO_MODEL / AI_IMAGE_MODEL 覆盖。 + # + # **只有型号可配,请求形状不可配**:哪个模型吃 image_list、哪个吃 + # input_reference、FAL 队列路径长什么样,都是该模型的 API 事实而非运行参数, + # 写在 providers.sufy 的映射表里。放进配置会把"填错了会怎样"从部署期推到 + # 运行期 —— 字段塞错不会立刻报错,任务照常 queued,直到生成阶段才 failed, + # 而费用可能已经产生(2026-07-29 实测)。 + video_model: str = "kling-v2-5-turbo" + image_model: str = "gemini-2.5-flash-image" + @property def normalized_base_url(self) -> str: return self.base_url.rstrip("/") diff --git a/backend/packages/framework/src/windup_framework/providers/__init__.py b/backend/packages/framework/src/windup_framework/providers/__init__.py index 3524bbf3..fd1f448e 100644 --- a/backend/packages/framework/src/windup_framework/providers/__init__.py +++ b/backend/packages/framework/src/windup_framework/providers/__init__.py @@ -1,8 +1,18 @@ -"""按模型能力划分的 AI Provider 接口。""" +"""按模型能力划分的 AI Provider:官方客户端工厂 + 能力接口 + SUFY 实现。""" from windup_framework.config.provider import AIProviderSettings from windup_framework.providers.chat import create_chat_model from windup_framework.providers.image import create_image_client +from windup_framework.providers.interfaces import ( + ImageProvider, + MatteProvider, + VideoProvider, +) +from windup_framework.providers.matte import OnnxU2NetMatteProvider +from windup_framework.providers.sufy import ( + SufyImageProvider, + SufyVideoProvider, +) from windup_framework.providers.video import create_video_client __all__ = [ @@ -10,4 +20,13 @@ "create_chat_model", "create_image_client", "create_video_client", + # 能力接口(ai_engine 依赖这些稳定契约) + "ImageProvider", + "VideoProvider", + "MatteProvider", + # 实现 + "SufyVideoProvider", + # FAL 队列面的 i2v(现役接口形态);首帧要公网 URL,故与 uploader 成对出现 + "SufyImageProvider", + "OnnxU2NetMatteProvider", ] diff --git a/backend/packages/framework/src/windup_framework/providers/interfaces.py b/backend/packages/framework/src/windup_framework/providers/interfaces.py new file mode 100644 index 00000000..4addf0f0 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/providers/interfaces.py @@ -0,0 +1,43 @@ +"""AI 模型底层适配器接口(framework)—— behind interface,key 由 config 注入。 + +ai_engine 经这些接口调模型,不直接读 env、不锁死具体供应商 / 模型名(可 A/B 换)。 +实测在用:图像 = gemini-flash-image;视频 = kling-v2-5-turbo(2026-07-27 端到端实测 +到 completed;#53 早期"仅 o1 可用、v2-5-turbo 下架"的结论已被该实测推翻);抠图 = rembg。 + +本文件是接口契约(真);具体 HTTP 实现见 :mod:`.sufy`。 +""" +from __future__ import annotations + +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class ImageProvider(Protocol): + """文 + 参考图 → 图(视角规整 / 定妆 / 逐帧生成)。""" + + def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: ... + + +@runtime_checkable +class VideoProvider(Protocol): + """首帧图 + 动作 prompt → 视频(i2v,步态位移动作用)。 + + **入参恒为 bytes,不是 URL。** 上游(ai_engine.strategy)手里只有母版 bytes,而且它 + 必须有 bytes —— ``master_check`` 预检、``master_prep`` 预处理、像素化锁色板全都读 + 母版**像素**。让调用点改传 URL 的话,ai_engine 还得自己下载回 bytes 才能干活。 + + 某些供应商的接口只吃公网 URL。那属于**该 provider 自己的适配问题**:在 provider + 内部完成 bytes → URL 的转换(需要一个上传能力时由组装层注入),而不是把这个差异 + 漏给上层。这样"用哪个厂商"不会改变 ai_engine 的一行代码。 + """ + + def i2v( + self, first_frame: bytes, prompt: str, seconds: int = 5, size: str = "1280x720" + ) -> bytes: ... + + +@runtime_checkable +class MatteProvider(Protocol): + """主体抠图(rembg / u2net)—— 按主体抠,不抠颜色(浅色角色撞背景会抠穿)。""" + + def cutout(self, frame: bytes) -> bytes: ... diff --git a/backend/packages/framework/src/windup_framework/providers/matte.py b/backend/packages/framework/src/windup_framework/providers/matte.py new file mode 100644 index 00000000..5695316f --- /dev/null +++ b/backend/packages/framework/src/windup_framework/providers/matte.py @@ -0,0 +1,233 @@ +"""主体抠图 MatteProvider —— onnxruntime 直跑 u2netp,不依赖 rembg。 + +为什么不用 rembg:rembg → pymatting → numba 0.53 / llvmlite 0.36 这条老链在 Python +3.12 无轮子(实测装不上)。而 rembg 内核就是"u2netp.onnx 过一遍 onnxruntime";默认 +``alpha_matting=False`` 时根本不碰 pymatting。故直调 onnxruntime,甩掉整条死重依赖, +3.12 干净可装、可进 lock。同模型(u2netp),同质量。 + +模型解析顺序:显式 ``model_path`` → 缓存目录已存在 → 从 ``model_url`` 惰性下载。 +onnxruntime 惰性导入(启动慢、按需加载),会话按需构建一次。 +""" +from __future__ import annotations + +import io +import urllib.request +from pathlib import Path + +import numpy as np +from PIL import Image + +from .interfaces import MatteProvider + +# u2netp:轻量版(~4.7MB)。rembg 官方 release 托管;国内不可达时可预置 model_path。 +_U2NETP_URL = "https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2netp.onnx" +_DEFAULT_CACHE = Path.home() / ".cache" / "windup" / "u2netp.onnx" + +# u2net 预处理常量(与 rembg 一致)。 +_MEAN = (0.485, 0.456, 0.406) +_STD = (0.229, 0.224, 0.225) +_SIZE = (320, 320) + + +# 只清理"几乎精确等于底色"的像素。阈值必须窄:2026-08-07 实测,一个铁锈橙毛 +# (222,130,70)的角色配玫红底(222,41,124),两者红通道完全相同、欧氏距离仅 104 —— +# 宽阈值会把毛判成半透明并去"反解",越解越坏(先成橄榄绿再成亮绿)。橙毛 d≈117, +# 阈值 38 完全碰不到它;而闭合空隙里的背景 d≈0,能干净移除。 +_KEY_KILL = 38.0 # d < 此值 → 判为纯背景 +_KEY_SOFT = 14.0 # 到 _KEY_KILL + _KEY_SOFT 之间线性过渡,避免硬边锯齿 +_BG_FLAT_STD = 8.0 # 四角色标准差上限;超过说明底不是纯色,不做任何清理 + +# 采样前先丢掉最外圈像素。视频帧的最外一两行/列常是**编码器边缘伪影**,不是底色: +# 2026-08-10 实测 9 段真 i2v 视频 × 16 帧 = 144 帧,贴边采样时 26 帧(18%)判"底不均匀" +# 而跳过清理,逐一查证全部由最外圈造成 —— 白底母版视频最右一列整列纯黑(std 50.4), +# 待机视频最顶一行偏暗(std 8.4,恰好压线越过 8)。往里让 1 px 就降到 1.9、让 2 px 降到 1.88, +# 144 帧零误跳;三张静态母版的取样中位色一个字节都没变(220/64/135、222/39/130、222/41/124)。 +# 取 2 是为容下 2 px 宽的边框;真正不均匀的底(噪声/渐变/拼色)让多少都照样超阈值,守卫不松。 +_EDGE_SKIP = 2 +_CORNER = 12 # 每个角的采样块边长 + + +def _corner_pixels(rgb: np.ndarray) -> np.ndarray: + """四角采样块(跳过最外圈 ``_EDGE_SKIP`` 像素)拼成的 (N, 3) 像素表。 + + 图太小时(四角会互相重叠)不让,退回贴边取 —— 合成测试图和缩略图走这条路。 + """ + k = _CORNER + s = _EDGE_SKIP if min(rgb.shape[:2]) > 2 * (_EDGE_SKIP + k) else 0 + r = rgb[s : rgb.shape[0] - s, s : rgb.shape[1] - s] if s else rgb + return np.concatenate([ + r[:k, :k].reshape(-1, 3), r[:k, -k:].reshape(-1, 3), + r[-k:, :k].reshape(-1, 3), r[-k:, -k:].reshape(-1, 3), + ]) + + +# 空洞填充用。_HOLE_ALPHA:低于此 alpha 才算"透明",参与空洞判定。 +# _HOLE_BG_TOL:到底色的距离低于此值 → 判为"确实是底色"。取值依据(2026-08-11 实测, +# 1280×720 真实视频帧):纯背景区域的色距 p99.9≈6.5、最大 11.1(视频压缩噪点); +# 而被误杀的浅肤色像素连通域中位色距 ≥17.1。14 落在这条 1.5 倍间隙里。 +_HOLE_ALPHA = 0.03 +_HOLE_BG_TOL = 14.0 + + +def _bg_key(rgb: np.ndarray) -> np.ndarray | None: + """四角取样估底色 key;底不够均匀(std 超阈值)时返回 None = 不做任何基于底色的判断。 + + 抽成独立函数是为了让"底色是什么"只有一个真相源 —— 键控清理(``_flat_bg_penalty``) + 和空洞填充(``_fill_enclosed_holes``)必须按同一个 key 判断,否则一个把某块当背景 + 清掉、另一个又把它当主体填回来,互相打架。取样统一走 :func:`_corner_pixels`, + 连"跳过最外圈编码器伪影"这条也只有一份实现。 + """ + corners = _corner_pixels(rgb) # 跳过编码器边缘伪影,见 _EDGE_SKIP + if float(corners.std(axis=0).max()) > _BG_FLAT_STD: + return None + return np.median(corners, axis=0).astype(np.float32) + + +def _spread(seed: np.ndarray, region: np.ndarray) -> np.ndarray: + """在 ``region`` 内从 ``seed`` 出发做 4-邻接连通扩散,返回可达集合。 + + 为什么不写逐像素 BFS:交付前的帧是 1280×720(约 92 万像素),纯 Python BFS 要几十秒, + 抠图是逐帧调用的,扛不住。这里按**行/列游程**传播 —— 一个 pass 就能把可达性推过 + 整条连续游程(距离不限),而不是每 pass 只推进一个像素,真实角色轮廓几个 pass 收敛。 + + 同一行里被非 region 像素隔断的两段游程,``cumsum(~region)`` 必然取到不同的 id, + 因此可以用 ``bincount`` 一次算出"每条游程里有没有种子"。 + """ + reach = seed & region + while True: + before = int(reach.sum()) + for transposed in (False, True): + reg = region.T if transposed else region + rch = reach.T if transposed else reach + rows, cols = reg.shape + run = np.cumsum(~reg, axis=1) + keys = run + np.arange(rows)[:, None] * (cols + 1) + hit = np.bincount(keys[rch], minlength=rows * (cols + 1)) > 0 + new = reg & hit[keys] + reach = new.T if transposed else new + if int(reach.sum()) == before: + return reach + + +def _fill_enclosed_holes(alpha: np.ndarray, rgb: np.ndarray) -> np.ndarray: + """把"被主体围住、且整块都不是底色"的透明连通域填回主体(alpha=1)。 + + 要解决的问题:u2netp 判错或键控误杀会在主体内部留下透明洞,放大看是背景直接透出来。 + + **为什么只判"不与边界连通"不够 —— 会把两腿之间填实。** 直觉上腿间空隙从下方通到 + 画面底边,所以"从边界出发的连通域"就能保护它。2026-08-11 在真实走路帧上实测: + **不成立**。迈步相里两只靴子在下方交叠,把腿间空隙彻底封死 —— 它就是一块不与边界 + 连通的背景域(实测 src_017 有 530 像素、归档 frame_03 有 129 像素),只按连通性判, + 这一整块会被填成主体,两条腿直接焊在一起。 + + 所以判据是**连通性 + 颜色**两条一起:一个透明连通域只要"碰到画面边界"或者"里面 + 存在任何一个确实是底色的像素",就不是洞。腿间空隙整块就是底色(实测中位色距 6.2, + 远低于 _HOLE_BG_TOL),必然被这条否决;而被误杀的主体像素(实测中位色距 ≥17.1) + 不含底色像素,才会被填。两条否决合成一次扩散:种子 = 边界上的透明像素 ∪ 底色像素。 + + 与 ``_flat_bg_penalty`` 的分工:那个函数按颜色**做减法**(把闭合空隙里的底色清掉), + 这个函数按颜色**决定不加回来** —— 同一个 key、同一个方向,不会互相拆台。 + """ + key = _bg_key(rgb) + if key is None: + return alpha # 底不是纯色 → 无从判断哪块是真空隙,一律不填 + transparent = alpha < _HOLE_ALPHA + if not transparent.any(): + return alpha + border = np.zeros_like(transparent) + border[0, :] = border[-1, :] = True + border[:, 0] = border[:, -1] = True + is_bg_color = np.linalg.norm(rgb - key, axis=2) < _HOLE_BG_TOL + seed = transparent & (border | is_bg_color) + holes = transparent & ~_spread(seed, transparent) + if not holes.any(): + return alpha + out = alpha.copy() + out[holes] = 1.0 + return out + + +def _flat_bg_penalty(rgb: np.ndarray) -> np.ndarray: + """底色清理系数(0=纯背景,1=主体),形状与图同宽高。 + + 为什么需要它:u2netp 是显著性模型,对**闭合区域**天然失灵 —— 四足角色腿间的 + 背景是一块被主体围住的空隙,显著性把它当成主体内部,整块底色留在产物里 + (2026-08-07 实测)。而母版底色是刻意生成的纯色,均匀度极高(实测四角标准差 1.0~1.2), + 用它做一次窄阈值清理就能补上这个洞。 + + 与"按颜色抠是死路"那条规则的边界:那条说的是**拿颜色当主体判据**(白底浅色角色 + 会被抠穿)。这里主体判据仍然是 u2netp,颜色只用来**做减法** —— 绝不新增主体像素, + 最坏情况是少清理一点,不会抠穿角色。底色不够均匀时(std 超阈值)直接返回全 1, + 等于不清理。 + + **逐帧独立采样是安全的**(2026-08-10 在真视频帧上验证):同一段视频里逐帧算出的 key 色 + 几乎不动(9 段 i2v 实测帧间位移 <= 1.73/255),故不需要跨帧共享一次采样。序列帧真正的 + 闪烁源是**守卫在序列中途翻转**(部分帧清、部分帧不清):待机那段 16 帧里前 6 帧清、后 10 帧 + 不清,主体面积逐帧变化 CV 从 0.0036 跳到 0.0197、第 6 帧单帧跳 4.25%。跳过最外圈后 + 守卫不再翻转,CV 回到 0.0028 —— 比完全不清理还稳(清理同时抹掉了会自己抖的底色描边)。 + """ + key = _bg_key(rgb) + if key is None: + return np.ones(rgb.shape[:2], dtype=np.float32) # 底不是纯色 → 不动 + d = np.linalg.norm(rgb - key, axis=2) + return np.clip((d - _KEY_KILL) / _KEY_SOFT, 0.0, 1.0).astype(np.float32) + + +class OnnxU2NetMatteProvider(MatteProvider): + """u2netp.onnx via onnxruntime。frame bytes → 抠好的 PNG(RGBA) bytes。""" + + def __init__(self, model_path: str | Path | None = None, model_url: str = _U2NETP_URL) -> None: + self._model_path = Path(model_path) if model_path else _DEFAULT_CACHE + self._model_url = model_url + self._session = None # 惰性 + + def _ensure_model(self) -> Path: + if not self._model_path.exists(): + self._model_path.parent.mkdir(parents=True, exist_ok=True) + urllib.request.urlretrieve(self._model_url, self._model_path) + return self._model_path + + def _get_session(self): + if self._session is None: + try: + import onnxruntime as ort # 惰性:导入慢 + except ImportError as e: # pragma: no cover - 取决于安装环境 + # **不静默降级。** 这里曾在 ImportError 时回落到"取四角主色做 chroma-key", + # 有两个问题:①猜背景色 —— 白底母版四角就是白色,浅色角色(骨白/银甲)与背景 + # 撞色会被抠穿;②静默 —— 开发机上看着能跑、输出其实是坏的,要到产物验收才发现。 + raise RuntimeError( + "onnxruntime 不可用，无法做主体抠图。请安装 onnxruntime" + "（注意 <1.24 才有 macOS Intel 轮子）。" + ) from e + self._session = ort.InferenceSession( + str(self._ensure_model()), providers=["CPUExecutionProvider"] + ) + return self._session + + def _predict_mask(self, img: Image.Image) -> Image.Image: + """u2netp 前向 → 单通道显著性 mask(L,原图尺寸)。""" + im = img.convert("RGB").resize(_SIZE, Image.LANCZOS) + ary = np.array(im).astype(np.float32) + ary = ary / max(float(ary.max()), 1e-6) + tmp = np.zeros((_SIZE[1], _SIZE[0], 3), dtype=np.float32) + for c in range(3): + tmp[:, :, c] = (ary[:, :, c] - _MEAN[c]) / _STD[c] + tensor = np.expand_dims(tmp.transpose(2, 0, 1), 0).astype(np.float32) + + session = self._get_session() + pred = session.run(None, {session.get_inputs()[0].name: tensor})[0][:, 0, :, :] + mi, ma = float(pred.min()), float(pred.max()) + pred = (pred - mi) / max(ma - mi, 1e-6) + mask = (pred.squeeze() * 255).astype(np.uint8) + return Image.fromarray(mask, "L").resize(img.size, Image.LANCZOS) + + def cutout(self, frame: bytes) -> bytes: + img = Image.open(io.BytesIO(frame)).convert("RGBA") + rgb = np.asarray(img.convert("RGB"), dtype=np.float32) + alpha = np.asarray(self._predict_mask(img), dtype=np.float32) / 255.0 + alpha = alpha * _flat_bg_penalty(rgb) + alpha = _fill_enclosed_holes(alpha, rgb) + out = np.dstack([np.asarray(img.convert("RGB")), alpha * 255.0]).astype(np.uint8) + buf = io.BytesIO() + Image.fromarray(out, "RGBA").save(buf, "PNG") + return buf.getvalue() diff --git a/backend/packages/framework/src/windup_framework/providers/sufy.py b/backend/packages/framework/src/windup_framework/providers/sufy.py new file mode 100644 index 00000000..31ec80f7 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/providers/sufy.py @@ -0,0 +1,357 @@ +"""Provider 接口的 SUFY / qnaigc(Modelink 网关)同步实现。 + +本模块实现三个 provider:视频(i2v)、图像(文生图 / 图生图)、以及它们共用的下载与首帧 +处理。抠图另在 :mod:`.matte`。 + +视频走 OpenAI 风格面(:class:`SufyVideoProvider`),首帧是 base64 dataURI:: + + POST /v1/videos {model, prompt, size, seconds, mode, input_reference} + 轮询 GET /v1/videos/{id} → status==completed → task_result.videos[0].url → 下载 mp4 + +2026-07-27 对 kling-v2-5-turbo 端到端实测到 completed。 + +图像走 OpenAI 兼容的 ``/chat/completions``(:class:`SufyImageProvider`),参考图以 data URI +塞进 ``content`` 数组 —— 与视频的提交-轮询-下载三段式完全不同的调用形状。 + +**网关上还有另一套 FAL 队列面**(veo / seedance / vidu 只在那一面)。曾实现过,但因为 +从未被真实调用过而移除,见本文件中段那条注释里记下的两个实测事实。 + +型号与 key / base_url 均由 ``AIProviderSettings`` 注入,provider 内不读 env;哪个模型吃 +什么请求字段属该模型的 API 事实,写在代码里而不是配置里(填错只会在生成阶段才 failed, +而费用可能已产生)。重依赖(PIL)惰性导入,保证模块导入零成本。 +""" +from __future__ import annotations + +import base64 +import io +import json +import logging +import re +import time + +import httpx + +from windup_framework.config.provider import AIProviderSettings, settings + +from .interfaces import ImageProvider, VideoProvider + +logger = logging.getLogger("windup.providers.sufy") + +# 只有 kling-video-o1 走 image_list;v2 系列 / sora 走 input_reference(字段按模型选,塞错任务会 failed)。 +_IMAGE_LIST_MODELS = ("kling-video-o1",) +DEFAULT_VIDEO_MODEL = "kling-v2-5-turbo" + + +def _fit_first_frame(frame: bytes, size: str) -> bytes: + """首帧 bytes → 等比缩放 + 背景色补边到目标尺寸 → JPG(RGB,q90) bytes。 + + 不强拉到目标尺寸(母版多为横幅,强压成方会把角色压成瘦长鬼影);JPG 因 PNG base64 + 会 VENDOR_FAILED(实测)。 + + 这一步同时是 kling 系"输出画幅"的唯一控制点:kling 的 i2v 端点没有 resolution/size + 字段,成片画幅跟随首帧,所以 ``size`` 只能在这里生效。 + """ + from PIL import Image + + w, h = (int(x) for x in size.split("x")) + im = Image.open(io.BytesIO(frame)).convert("RGB") + pad = im.getpixel((0, 0)) + fitted = im.copy() + fitted.thumbnail((w, h), Image.LANCZOS) + canvas = Image.new("RGB", (w, h), pad) + canvas.paste(fitted, ((w - fitted.width) // 2, (h - fitted.height) // 2)) + buf = io.BytesIO() + canvas.save(buf, "JPEG", quality=90) + return buf.getvalue() + + +def _first_frame_datauri(frame: bytes, size: str) -> str: + """首帧 → base64 dataURI(OpenAI 风格 ``/v1/videos`` 面专用;FAL 面不吃 dataURI)。""" + return "data:image/jpeg;base64," + base64.b64encode(_fit_first_frame(frame, size)).decode() + + +class SufyVideoProvider(VideoProvider): + """kling i2v(默认 v2-5-turbo)。首帧 + 动作 prompt → mp4 bytes。""" + + def __init__( + self, + config: AIProviderSettings = settings, + model: str | None = None, + mode: str = "std", + poll_interval: float = 60.0, + max_min: int = 30, + ) -> None: + # 轮询间隔必须 > 0:下面用 `max_min * 60 // poll` 算预算次数,传 0 直接除零 + # (2026-08-11 补 i2v 主流程测试时逮到)。0 的语义本身也不成立 —— 那是忙等, + # 会把网关打满。测试要跑快就把 time.sleep 打桩掉,别把间隔设成 0。 + if poll_interval <= 0: + raise ValueError(f"poll_interval 必须为正数,收到 {poll_interval}") + self._cfg = config + self._model = model or config.video_model + self._mode = mode + self._poll = poll_interval + self._max_min = max_min + + def _client(self) -> httpx.Client: + return httpx.Client( + base_url=self._cfg.normalized_base_url, + headers={"Authorization": f"Bearer {self._cfg.api_key}"}, + timeout=self._cfg.timeout, + ) + + def i2v( + self, first_frame: bytes, prompt: str, seconds: int = 5, size: str = "1280x720" + ) -> bytes: + body: dict = { + "model": self._model, + "prompt": prompt, + "size": size, + "seconds": str(seconds), + "mode": self._mode, + } + if self._model in _IMAGE_LIST_MODELS: + b64 = _first_frame_datauri(first_frame, size).split(",", 1)[1] + body["image_list"] = [{"image": b64}] + else: + body["input_reference"] = _first_frame_datauri(first_frame, size) + + with self._client() as client: + job = client.post("/videos", json=body).raise_for_status().json() + jid = job.get("id") + url = None + for _ in range(max(1, int(self._max_min * 60 // self._poll))): + time.sleep(self._poll) + st = client.get(f"/videos/{jid}").raise_for_status().json() + status = st.get("status") + if status == "completed": + vids = (st.get("task_result") or {}).get("videos") or [] + url = vids[0].get("url") if vids else None + break + if status in ("failed", "cancelled"): + raise RuntimeError(f"i2v 失败: {status} — {st.get('error')}") + if not url: + raise RuntimeError("i2v 未取得视频 URL(超时或失败)") + return _download(client, url) + + +class IncompleteDownloadError(RuntimeError): + """视频下载到的字节数与 ``Content-Length`` 不符。""" + + +class UnsafeDownloadUrlError(RuntimeError): + """成品 URL 的协议不是 http(s) —— 不下载。 + + 这个 URL 来自网关响应,是外部输入。直接丢给 httpx 去 GET 一个 ``file://`` / ``data:`` + 只会在重试三次之后报一个跟协议无关的传输错,不如在这里就说清是地址不对。 + """ + + +def _same_origin(url: httpx.URL, other: httpx.URL) -> bool: + """同源判定(scheme + host + 端口,默认端口按 scheme 补齐)。 + + 语义对齐 httpx 自己在跨源重定向时摘凭证用的 ``Client._redirect_headers``; + 没直接 import 它的私有 ``_same_origin``,免得被上游改名。 + + "默认端口补齐"这一步在 httpx 0.28 下其实判不出新差别(它已把 ``:443`` / ``:80`` + 归一化成 ``port is None``,2026-08-10 变异测试确认单独拆掉这行无用例失败)。留着的理由 + 是与 httpx 保持同一套判据:一旦上游不再归一化,少了它 ``https://gw`` 与 ``https://gw:443`` + 就成了跨源,会把该带的凭证摘掉、把同源下载打成 401。 + """ + default = {"http": 80, "https": 443} + return ( + url.scheme == other.scheme + and url.host == other.host + and (url.port or default.get(url.scheme)) == (other.port or default.get(other.scheme)) + ) + + +def _download_request(client: httpx.Client, url: str) -> httpx.Request: + """构造成品下载请求;目标不在网关同源时,把 client 级凭证摘掉。 + + 为什么必须摘(2026-08-10 机器审提出):成品 URL 是**网关响应里的绝对地址**,正常情况 + 指向 CDN 域名,异常情况可以是网关返回的任意地址。而 httpx 只在跨源**重定向**时才自动 + 摘 Authorization,对这种一开始就跨源的直连请求,client 级 headers 会原样带过去 —— + 于是 ``Authorization: Bearer/Key `` 被发给了那个域名,等于把 API key 交出去。 + + 同源时保留凭证:网关也可能签发自己域名下的下载链接,那条路径摘了头就是 401。 + 所以按目标地址判定,不是一律摘、也不是一律留。 + """ + request = client.build_request("GET", url) + if request.url.scheme not in ("http", "https"): + raise UnsafeDownloadUrlError(f"成品 URL 必须是 http(s),收到 {str(request.url)!r}") + if not _same_origin(request.url, client.base_url): + # 只摘目标域名不该看到的:Proxy-Authorization 是给代理的,与目标是否同源无关,别动它。 + request.headers.pop("Authorization", None) + request.headers.pop("Cookie", None) + return request + + +def _download(client: httpx.Client, url: str, tries: int = 3) -> bytes: + """下载已生成好的视频,带重试 + 长度校验。 + + 为什么单次读取不够(2026-08-05 实测,同一角色连续两单复现):原实现是 + ``client.get(url).raise_for_status().content``。**视频此时已经生成、费用已经产生**, + 只要读 body 时连接断一次,整单就废:: + + peer closed connection without sending complete message body + (received 720450 bytes, expected 929531) + + 重试是安全的:这是对成品 URL 的 GET,幂等且不再计费——**代价是一次重下, + 不重试的代价是一次重新生成**。 + + 长度校验是因为截断不一定抛异常:服务端提前关流而客户端已收到部分 body 时, + ``.content`` 可能直接返回短 bytes,那样坏视频会一路流到出帧环节才暴露, + 在那里看起来像"解码失败",很难回溯到这里。``Content-Length`` 缺失(分块传输)时跳过校验。 + + 凭证处理见 :func:`_download_request`。请求在进循环之前就构造好:地址不合法要在 + 发出任何一次请求之前炸,而不是重试三次之后。 + """ + request = _download_request(client, url) + last: Exception | None = None + for attempt in range(tries): + try: + # send 不会再合并 client 级 headers(build_request 时已合并过), + # 所以上面摘掉的 Authorization 不会被重新加回来。 + response = client.send(request) + response.raise_for_status() + body = response.content + expected = response.headers.get("content-length") + if expected and len(body) != int(expected): + raise IncompleteDownloadError(f"视频下载不完整: {len(body)}/{expected} 字节") + return body + except (httpx.HTTPError, IncompleteDownloadError) as exc: + last = exc + if attempt < tries - 1: + time.sleep(2**attempt) + raise RuntimeError(f"视频下载失败(已重试 {tries} 次): {last}") from last + + +# ── FAL 队列面 ────────────────────────────────────────────────────────────── +# 2026-08-07 拉网关 OpenAPI spec 核对得到:平台的 22 个图生视频端点全在 /queue/ 下, +# 首帧字段一律是 URL 形态(image_url / start_image_url),同日实测送 dataURI 无一能用。 +# (spec 里 seedance / vidu-q3 / kling-v3-turbo 三家的字段说明写着"URL 或 base64", +# 与实测冲突,未复验。本实现一律只发公网 URL —— 那是 22 个端点的共同解。) +# +# 每家有三样东西不一样,而且**没有一条能靠拼字符串猜出来**,所以下面是一张硬表: +# 1. 提交路径:型号段各不相同(o3 / v3 / v3/turbo / v2.6 / v2.5-turbo / o1), +# 有的带 {mode} 路径参数、有的不带(veo / seedance / minimax / vidu 不带)。 +# 2. 首帧字段名:同是 kling,o3 与 v2.5-turbo 叫 image_url,v3 / v2.6 / o1 却叫 +# start_image_url。塞错字段 = 送了图但模型没收到。 +# 3. 轮询前缀:**不是**提交路径加个 /requests。kling 六个型号共用一个 +# /queue/fal-ai/kling-video/requests/{id},型号段与 mode 段都不出现。 +# 这一条是最容易想当然拼错的地方。 +# +# 另有两处形态差异也写进表里,因为取值形式不同会被网关 400: +# - 时长字段都叫 duration,但取值分三种形态:"5"(kling/seedance)、"8s"(veo)、 +# 5(minimax/vidu,整数)。 +# - 分辨率:kling 系**没有**这个字段(成片画幅跟随首帧,所以 size 只能靠补边生效); +# 其余各家的档位枚举各不相同。 + + +# ── FAL 队列面（veo / seedance / vidu）已移除 ──────────────────────────────── +# +# 曾有一整套 FalQueueVideoProvider + FirstFrameUploader + 端点映射表（412 行、28 条 +# 测试）。删掉的理由与 GenRoute 只列有实现的路线是同一条：**它从未被真实调用过** +# —— app / ai_engine 里零引用，产品链路走不到，而"代码在仓里"会让人以为该能力已具备。 +# +# 真要接 veo / seedance 时连同一次真实调用一起加回。届时的两个已知事实（实测挣得， +# 别再摸索一遍）： +# 1. FAL 面只吃**公网 URL**，不吃 base64；塞 base64 会 status=queued 之后在生成阶段 +# 才 failed，费用可能已经产生。 +# 2. 鉴权头是 `Authorization: Key `，不是 `Bearer`；路径与 /v1 平级，不是它的子路径。 +# 归档实测记录见项目参考资料（图生视频 API 实测文档）。 + + +DEFAULT_IMAGE_MODEL = "gemini-2.5-flash-image" + +# "调用成功但没返回有效图"的重试次数。与 _download 的网络重试是两码事:那个治连接断, +# 这个治模型返回了一条不含图的正常响应(实测偶发)。也是为什么下面要判 base64 长度 —— +# 返回里可能带一个几十字节的占位串,当图存下去就是一个打不开的文件。 +_IMAGE_TRIES = 3 +_MIN_IMAGE_BYTES = 5000 +_CONNECT_RETRIES = 3 + +# 从响应里捞 data URI。模型把图放在 message.content 里,而不同网关的包裹层级不一样 +# (有的 content 是字符串、有的是 parts 数组),故对整个响应 JSON 做一次正则, +# 不去猜层级 —— 猜错的代价是"调用成功、费用已产生、但我们说没图"。 +_DATA_URI = re.compile(r"data:image/[^;]+;base64,([A-Za-z0-9+/=]{100,})") + + +class SufyImageProvider(ImageProvider): + """文生图 / 图生图 provider(OpenAI 兼容的 ``/chat/completions`` 面)。 + + 调用形状与 i2v 那两个 provider 完全不同:图像走 chat 接口、参考图以 data URI 塞进 + ``content`` 数组,没有提交-轮询-下载三段式。 + + 2026-08-10 修:此前 ``gen_image`` 直接抛 NotImplementedError,而 + ``POST /generation/image`` 端点是可达的、``ImageTaskExecutor`` 又默认实例化本类 —— + 于是每个图像任务都稳定走到 FAILED。端点看着可用、实际必失败,正是本仓最忌讳的形态 + (机器审逮到)。实现取自管线仓已跑通的通路(同日用它出过三张角色母版)。 + """ + + def __init__( + self, + config: AIProviderSettings = settings, + model: str | None = None, + ) -> None: + self._cfg = config + self._model = model or config.image_model + + def _client(self) -> httpx.Client: + return httpx.Client( + base_url=self._cfg.normalized_base_url, + headers={"Authorization": f"Bearer {self._cfg.api_key}"}, + timeout=self._cfg.timeout, + # retries 只覆盖建连阶段的失败(SSL 握手、连接被重置)。本机走代理时这类抖动 + # 常见,已跑通的管线实现正是靠一层网络重试扛住的;不加会在人家能恢复的地方 + # 放弃。它不重试读超时与 5xx —— 那两种请求可能已达上游,重发会重复计费。 + transport=httpx.HTTPTransport(retries=_CONNECT_RETRIES), + ) + + def _post(self, client: httpx.Client, body: dict) -> dict: + """发一次请求。把"网关没有这个模型"翻译成能照着修的错误。 + + 为什么值得专门处理:同一把 key 下不同网关的模型目录**不一样**。实测 + ``GET /v1/models``:一个网关 73 个模型、一个图像模型都没有;另一个 134 个、 + 含本模块默认的那个(2026-08-10)。配错 ``AI_BASE_URL`` 时原始报错只是一条 + 404,读的人无从知道该去改配置还是改模型名。 + """ + resp = client.post(self._cfg.chat_completions_path, json=body) + if resp.status_code in (400, 404): + raise RuntimeError( + f"网关 {self._cfg.normalized_base_url} 拒绝了模型 {self._model!r}" + f"(HTTP {resp.status_code})。先确认该网关的目录里有它:" + f"GET {self._cfg.normalized_base_url}/models —— 不同网关目录不同," + f"同一把 key 也是。原始响应:{resp.text[:200]}" + ) + return resp.raise_for_status().json() + + def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: + """提示词 + 参考图 → 一张 PNG bytes。拿不到有效图就抛,不返回空 bytes。 + + 为什么不返回空 bytes 兜底:上游 ``ImageTaskExecutor`` 会把返回值直接上传对象存储 + 并写进任务结果,一个 0 字节的"成功"会变成用户看到的一张裂图。 + """ + content: list[dict] = [{"type": "text", "text": prompt}] + for raw in refs: + b64 = base64.b64encode(raw).decode() + content.append({ + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + }) + body = {"model": self._model, "messages": [{"role": "user", "content": content}]} + + last = "" + with self._client() as client: + for attempt in range(1, _IMAGE_TRIES + 1): + payload = self._post(client, body) + found = _DATA_URI.search(json.dumps(payload)) + if found: + data = base64.b64decode(found.group(1)) + if len(data) >= _MIN_IMAGE_BYTES: + return data + last = f"图只有 {len(data)} 字节(下限 {_MIN_IMAGE_BYTES})" + else: + last = "响应里没有 data URI" + logger.warning("文生图第 %d/%d 次没拿到有效图:%s", attempt, _IMAGE_TRIES, last) + raise RuntimeError(f"文生图 {_IMAGE_TRIES} 次均未取得有效图:{last}") diff --git a/backend/tests/test_ai_engine_skeleton.py b/backend/tests/test_ai_engine_skeleton.py new file mode 100644 index 00000000..91e8ca2d --- /dev/null +++ b/backend/tests/test_ai_engine_skeleton.py @@ -0,0 +1,418 @@ +"""ai_engine 串联 smoke —— 验证架构串联成立:路由正确 + generate 端到端跑通。 + +策略内部(真实 i2v)用 mock / monkeypatch 顶替(真实生成联网、抽帧要解码 mp4); +本测证明"选路线 → derive → 最后一公里(真实对齐)→ GeneratedAction(帧 + 时长)"这条串联为真。 +""" +from __future__ import annotations + +import io + +from PIL import Image + +from windup_ai_engine.impl import CharacterGenerator +from windup_ai_engine.ports import GeneratedAction +from windup_ai_engine.postprocess.rootmotion import DEFAULT_FPS_MS +from windup_ai_engine.strategy import ( + ROUTE_MATRIX, + DerivationStrategy, + VideoFrameStrategy, +) +from windup_common.models import ( + ActionSpec, + ActionType, + CharacterCard, + Facing, + GenRoute, + Stylize, +) + + +def _tiny_png(color=(200, 60, 60, 255), shift=0) -> bytes: + """一张带主体的小 RGBA PNG(四周留透明边,供真实对齐 / 抠图链处理)。""" + img = Image.new("RGBA", (64, 96), (0, 0, 0, 0)) + for y in range(20, 80): + for x in range(24 + shift, 40 + shift): + img.putpixel((x, y), color) + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + +class _NullProgress: + def step(self, stage: str, i: int, total: int, note: str = "") -> None: + pass + + +class _MockWalkStrategy(DerivationStrategy): + """顶替真实 VideoFrameStrategy:返回 N 张真 PNG,让对齐真跑。""" + + route = GenRoute.VIDEO_I2V + + def derive(self, card, action, master, progress) -> list[bytes]: + return [_tiny_png() for _ in range(action.n_frames)] + + +def _make_generator() -> CharacterGenerator: + return CharacterGenerator({GenRoute.VIDEO_I2V: _MockWalkStrategy()}) + + +def test_route_matrix_is_the_measured_contract(): + # 实测挣得的架构决策:走路/跑/攻击走视频,受击逐帧,待机程序化 + assert ROUTE_MATRIX[ActionType.WALK] is GenRoute.VIDEO_I2V + assert ROUTE_MATRIX[ActionType.RUN] is GenRoute.VIDEO_I2V + assert ROUTE_MATRIX[ActionType.ATTACK] is GenRoute.VIDEO_I2V + assert ROUTE_MATRIX[ActionType.JUMP] is GenRoute.VIDEO_I2V + assert ROUTE_MATRIX[ActionType.HIT] is GenRoute.PER_FRAME + assert ROUTE_MATRIX[ActionType.IDLE] is GenRoute.VIDEO_I2V + + +def test_generate_walk_is_wired_end_to_end(): + card = CharacterCard(name="rogue", desc="hooded ranger, dual daggers") + # 视频路线只需声明帧数;poses 是逐帧路线的入参,这里不传(以前必须编 8 条假描述)。 + action = ActionSpec(action=ActionType.WALK, n_frames=8) + out = _make_generator().generate(card, action, master=_tiny_png(), progress=_NullProgress()) + assert isinstance(out, GeneratedAction) + assert len(out.frames) == 8 # 选路线→derive→对齐 全串通 + assert len(out.durations) == 8 # 逐帧时长与帧等长 + # 时长是按动作查表来的,不是从入参帧率算的 —— walk 的基准是 125ms/帧。 + # 原先这里断言的是 `out.fps == action.fps`,把"照抄一个不生效的入参"锁成了契约。 + assert all(d > 0 for d in out.durations) + assert set(out.durations) == {DEFAULT_FPS_MS["walk"]} + assert all(f and f[:8] == b"\x89PNG\r\n\x1a\n" for f in out.frames) # 真 PNG + + +def test_action_spec_stylize_defaults_and_toggle(): + # 像素化是开关(默认 pixel),可关成 none 保留 i2v 画风 + assert ActionSpec(action=ActionType.WALK).stylize is Stylize.PIXEL + a = ActionSpec(action=ActionType.WALK, stylize="none") + assert a.stylize is Stylize.NONE + + +def _offline_video_strategy(monkeypatch, video=None) -> VideoFrameStrategy: + """离线版 VideoFrameStrategy:抽帧被顶替,不解码 mp4 / 不联网 / 不花钱。""" + dense = [Image.open(io.BytesIO(_tiny_png(shift=i % 6))).convert("RGBA") for i in range(24)] + monkeypatch.setattr( + "windup_ai_engine.strategy.concrete.extract_all_frames_bytes", + lambda video, cap=150: dense, + ) + + class _StubVideo: + def i2v(self, first_frame, prompt, seconds=5, size="1280x720"): + return b"fake-mp4" + + class _StubMatte: + def cutout(self, frame): # 透传:合成帧已带 alpha + return frame + + return VideoFrameStrategy(video or _StubVideo(), _StubMatte()) + + +def test_video_strategy_derive_runs_offline(monkeypatch): + """真实 VideoFrameStrategy.derive 离线跑通。 + + 证明 derive 的真实链路:i2v → 抽帧 → 抠图 → 选帧 → 出帧,产物是合法 RGBA PNG。 + """ + strat = _offline_video_strategy(monkeypatch) + card = CharacterCard(name="knight", desc="plate armor, sword") + action = ActionSpec(action=ActionType.WALK, stylize="none", n_frames=8) + out = strat.derive(card, action, master=_tiny_png(), progress=_NullProgress()) + assert out and all(f[:8] == b"\x89PNG\r\n\x1a\n" for f in out) + + +def test_video_strategy_honours_n_frames_without_any_poses(monkeypatch): + """帧数由 ActionSpec.n_frames 决定,**不必传 poses** —— A2 的落地验证。 + + 以前只能靠 len(poses) 表达帧数,于是"要 6 帧"得先编 6 条视频路线根本不读的姿势描述; + 读代码的人会以为那 6 条描述真的进了提示词。 + """ + strat = _offline_video_strategy(monkeypatch) + card = CharacterCard(name="knight", desc="plate armor, sword") + for n in (4, 6, 11): + action = ActionSpec(action=ActionType.WALK, stylize="none", n_frames=n) + out = strat.derive(card, action, master=_tiny_png(), progress=_NullProgress()) + assert len(out) == n, f"要 {n} 帧,实得 {len(out)} 帧" + + +def test_video_strategy_stylize_switch_actually_changes_the_pixels(monkeypatch): + """stylize 分支不能接反 —— 只验"两条分支都不抛错"验不出接反。 + + none=原样出帧(与输入同尺寸);pixel=裁包围盒 + 重采样到目标像素高,尺寸必然不同。 + """ + strat = _offline_video_strategy(monkeypatch) + card = CharacterCard(name="knight", desc="plate armor, sword") + plain = strat.derive( + card, ActionSpec(action=ActionType.WALK, stylize=Stylize.NONE, n_frames=4), + master=_tiny_png(), progress=_NullProgress(), + ) + pixel = strat.derive( + card, ActionSpec(action=ActionType.WALK, stylize=Stylize.PIXEL, n_frames=4), + master=_tiny_png(), progress=_NullProgress(), + ) + assert Image.open(io.BytesIO(plain[0])).size == (64, 96) # 未像素化:原尺寸 + assert Image.open(io.BytesIO(pixel[0])).size != (64, 96) # 像素化:重采样过 + + +def test_video_strategy_prompt_follows_facing(monkeypatch): + """喂给 i2v 的提示词随 ActionSpec.facing 走 —— 朝向约束真的传到了付费调用那一层。 + + 这是 facing 枚举化要保护的东西:枚举保证值合法,本测保证合法值被用对。 + """ + seen: list[str] = [] + + class _SpyVideo: + def i2v(self, first_frame, prompt, seconds=5, size="1280x720"): + seen.append(prompt) + return b"fake-mp4" + + strat = _offline_video_strategy(monkeypatch, video=_SpyVideo()) + card = CharacterCard(name="knight", desc="plate armor, sword") + for facing in (Facing.SIDE, Facing.FRONT): + strat.derive( + card, + ActionSpec(action=ActionType.WALK, stylize=Stylize.NONE, n_frames=4, facing=facing), + master=_tiny_png(), progress=_NullProgress(), + ) + assert "SIDE VIEW facing right" in seen[0] + assert "FACING THE VIEWER" in seen[1] + + +def test_real_video_strategy_is_registered_for_video_route(): + # 真实 VideoFrameStrategy 可构造且声明视频路线(derive 联网,不在此跑) + class _V: + def i2v(self, first_frame, prompt, seconds=5, size="1280x720"): + return b"" + + class _M: + def cutout(self, frame): + return frame + + strat = VideoFrameStrategy(_V(), _M()) + assert strat.route is GenRoute.VIDEO_I2V + + +# ── 未实现的路线必须炸,不能吐空帧(2026-08-07)───────────────────────────── +# +# 旧行为:PerFrameStrategy.derive 返回 [b""] * n_frames,CharacterGenerator._lastmile +# 见到空帧就静默跳过对齐、原样返回。调用方拿到的 GeneratedAction 帧数对、时长对、 +# 无异常 —— 完全像一次成功的生成。server 会把 N 个 0 字节文件传上对象存储、写进 +# character_data,用户看到 N 张裂图,且排查时不会想到是"路线没实现"。 +# +# 新行为:在最早能判定的边界上抛错。下面三条分别覆盖三个入口。 + + +def test_unimplemented_route_raises_instead_of_returning_empty_frames(): + """PerFrameStrategy 调用即抛,不返回空帧。""" + import pytest + + from windup_ai_engine.strategy import PerFrameStrategy + + s = PerFrameStrategy(image=None, matte=None) + card = CharacterCard(name="t", desc="t") + action = ActionSpec(action=ActionType.HIT, poses=["a", "b", "c"]) + with pytest.raises(NotImplementedError, match="per_frame"): + s.derive(card, action, _tiny_png(), _NullProgress()) + + +def test_missing_strategy_for_route_raises_with_what_is_wired(): + """装配表里没有该路线时抛错,并报出已装配了哪些 —— 便于定位是漏注入还是没实现。""" + import pytest + + # 只装 VIDEO_I2V,请求 hit(分流到 PER_FRAME) + gen = CharacterGenerator({GenRoute.VIDEO_I2V: _MockWalkStrategy()}) + card = CharacterCard(name="t", desc="t") + action = ActionSpec(action=ActionType.HIT, poses=["a", "b"]) + with pytest.raises(NotImplementedError, match="video_i2v"): + gen.generate(card, action, _tiny_png(), _NullProgress()) + + +def test_empty_frames_from_strategy_are_rejected(): + """strategy 吐出空帧(provider / 抠图坏了)时同样要炸,不原样放行。""" + import pytest + + class _EmptyStrategy(DerivationStrategy): + route = GenRoute.VIDEO_I2V + + def derive(self, card, action, master, progress) -> list[bytes]: + return [b"", b"", b""] + + gen = CharacterGenerator({GenRoute.VIDEO_I2V: _EmptyStrategy()}) + card = CharacterCard(name="t", desc="t") + action = ActionSpec(action=ActionType.WALK, poses=["a", "b", "c"]) + with pytest.raises(ValueError, match="空帧"): + gen.generate(card, action, _tiny_png(), _NullProgress()) + + +def test_short_frame_count_from_strategy_is_rejected(): + """产出帧数少于 ``n_frames`` 时要炸 —— 少给不会崩,只会"短一截"。 + + 这不是假想:slicing.pick_cycle / pick_oneshot 在源帧不足(i2v 视频太短 / 动作区间 + 过窄)时 ``return frames`` / ``return span``,长度不足且不报错。时长表由 + frame_durations(…, len(frames)) 现算,所以产物内部自洽 —— server 看不出异常, + 用户拿到一段步子没走完的循环。A2 之后 n_frames 是调用方的明确承诺,必须对账。 + """ + import pytest + + class _ShortStrategy(DerivationStrategy): + route = GenRoute.VIDEO_I2V + + def derive(self, card, action, master, progress) -> list[bytes]: + return [_tiny_png() for _ in range(action.n_frames - 1)] # 少给一帧 + + gen = CharacterGenerator({GenRoute.VIDEO_I2V: _ShortStrategy()}) + card = CharacterCard(name="t", desc="t") + with pytest.raises(ValueError, match="要 8 帧,实际产出 7 帧"): + gen.generate( + card, ActionSpec(action=ActionType.WALK, n_frames=8), + _tiny_png(), _NullProgress(), + ) + + +def test_progress_notes_carry_enum_values_not_python_reprs(): + """进度文案里不能出现 "ActionType.WALK"。 + + Python 3.11 改了 str-mixin 枚举的 __format__:f"{ActionType.WALK}" 从 "walk" 变成 + "ActionType.WALK"(3.12.13 实测)。这串字经 server 变成用户看到的 SSE 进度文案, + 没有任何测试会因此变红 —— 属于"跑得通但对外是错的"那一类。 + """ + notes: list[str] = [] + + class _SpyProgress: + def step(self, stage: str, i: int, total: int, note: str = "") -> None: + notes.append(note) + + _make_generator().generate( + CharacterCard(name="t", desc="t"), + ActionSpec(action=ActionType.WALK, n_frames=4), + _tiny_png(), _SpyProgress(), + ) + assert notes, "没收到任何进度上报" + assert not any("ActionType." in n for n in notes), notes + assert any("walk" in n for n in notes), notes + + +def test_generated_action_has_a_single_timing_source(): + """出参不许有第二个描述播放速度的字段。 + + 此处曾有一条 ``test_loop_mode_currently_changes_nothing``,把"传 pingpong / none + 不改变任何一帧"钉成可执行事实,理由是"将来真接线时它会变红提醒删注释"。 + 那是把缺陷固化:调用方能为一段往返动画付费、拿到一段线性循环,而测试为这个行为背书。 + 2026-08-10 按机器审意见改成删字段 —— ``ActionSpec.loop`` 与 ``LoopMode`` 都已移除, + 真要支持 pingpong,连同 pick_cycle 的分支与出参时序契约一起加回。 + + 同批删掉的 ``GeneratedAction.fps`` 同理:它抄自入参、与 durations 互相矛盾。 + """ + from dataclasses import fields + + names = {f.name for f in fields(GeneratedAction)} + assert "fps" not in names, "fps 与 durations 会给出两个不同的播放速度" + assert "durations" in names + + +def test_genroute_only_lists_implemented_routes(): + """GenRoute 只列有实现的路线 —— 没有实现的枚举值等于死代码。 + + 这条同时管住两个方向： + - PROC_IDLE（程序化待机，#53 原设计）已证否，连同 ProcIdleStrategy 一并移除； + - 未来路线（三渲二渲染出帧）**不提前留位**，契约需求记在 Issue，随实现一起加成员。 + 枚举加成员是纯加法，不构成破坏性变更，所以"提前留位免得二次改形"不成立。 + """ + assert {r.value for r in GenRoute} == {"video_i2v", "per_frame"} + import windup_ai_engine.strategy as strat + assert not hasattr(strat, "ProcIdleStrategy") + + +# ── 交付画布尺寸(2026-08-11 挣得)──────────────────────────────────────────── +# +# 引擎此前恒出 256 方形,项目的 sprite 尺寸由上层再缩一次。那一步用 Image.thumbnail +# 补边,而 thumbnail **只缩不放**:项目要 512 时 256 的帧根本不会被放大,而是原尺寸 +# 居中贴进 512 画布,于是 align_bottom_center 刚对齐好的脚线 0.92 被挪到 0.709 +# (实测),角色不站在地上、跨动作对齐一并失效。故 canvas 直接传进引擎。 + + +def _delivered(png: bytes): + """返回 (画布尺寸, 主体高, 脚线比例)。""" + import numpy as np + + im = Image.open(io.BytesIO(png)).convert("RGBA") + ys, _ = np.nonzero(np.asarray(im)[:, :, 3] > 128) + return im.size, int(ys.max() - ys.min() + 1), (int(ys.max()) + 1) / im.height + + +def _run(canvas=None): + card = CharacterCard(name="rogue", desc="hooded ranger") + action = ActionSpec(action=ActionType.WALK, n_frames=4) + gen = _make_generator() + kw = {} if canvas is None else {"canvas": canvas} + return gen.generate(card, action, master=_tiny_png(), progress=_NullProgress(), **kw) + + +def test_canvas_omitted_keeps_the_256_default_byte_for_byte(): + """不传 canvas → 与加这个参数之前逐字节相同(默认行为不变)。""" + a = _run() + b = _run(canvas=(256, 256)) + assert [f for f in a.frames] == [f for f in b.frames] + assert _delivered(a.frames[0])[0] == (256, 256) + + +def test_canvas_512_doubles_delivered_subject_height(): + """指定 512 时交付帧主体高度约翻倍 —— 这就是"成品放大看很糊"的正解。""" + small = _delivered(_run().frames[0]) + big = _delivered(_run(canvas=(512, 512)).frames[0]) + assert big[0] == (512, 512) + assert abs(big[1] / small[1] - 2.0) < 0.05, f"期望约翻倍,实际 {small[1]} → {big[1]}" + + +def test_canvas_non_square_is_honoured_end_to_end(): + """非方形项目尺寸也要一次出到位:高度几何只看画布高,不被画布宽带偏。 + + 与同高的方形画布逐项比,而不是比一个算出来的期望值 —— 主体只有 60px 高, + 定标系数 5.4 倍,ref_height 上 1px 的取整差会被放大成 5px,拿绝对值卡阈值 + 量的是取整噪声不是行为。 + """ + tall, height_t, foot_t = _delivered(_run(canvas=(384, 512)).frames[0]) + square, height_s, foot_s = _delivered(_run(canvas=(512, 512)).frames[0]) + assert tall == (384, 512) and square == (512, 512) + assert height_t == height_s, "画布高相同 → 主体高必须相同(高度几何不看宽)" + assert foot_t == foot_s + assert abs(foot_t - 0.92) <= 0.01 + + +def test_canvas_reaches_every_frame_not_just_the_first(): + """整段每一帧都得是请求的画布 —— 半段没生效比不生效更难查。""" + out = _run(canvas=(320, 320)) + assert {_delivered(f)[0] for f in out.frames} == {(320, 320)} + + +def _wide_master(ratio: float) -> bytes: + """主体宽高比为 ratio 的母版(源画幅给足,别让主体被源边界裁掉)。""" + h = 60 + img = Image.new("RGBA", (int(h * ratio) + 200, 200), (0, 0, 0, 0)) + img.paste((200, 60, 60, 255), (20, 60, 20 + int(h * ratio), 60 + h)) + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + +def test_precheck_and_output_share_the_same_canvas_geometry(): + """入口预检必须按**出帧用的那个 canvas** 判,不能按方形判、按非方出。 + + 比例上限是从交付画布几何推出来的(REJECT_ASPECT*(cw/ch))。取一个夹在 + "方形阈值"与"384×512 阈值"之间的母版:方形画布下该放行,窄高画布下该拒。 + 两边不一致就说明预检和出帧用的不是同一套几何。 + """ + import pytest + + from windup_ai_engine.master_check import REJECT_ASPECT, reject_aspect_for + from windup_ai_engine.ports import MasterRejectCode, MasterRejected + + ratio = (REJECT_ASPECT + reject_aspect_for((384, 512))) / 2 + master = _wide_master(ratio) + card = CharacterCard(name="wide", desc="wide creature") + action = ActionSpec(action=ActionType.WALK, n_frames=2) + gen = _make_generator() + + gen.generate(card, action, master, _NullProgress(), canvas=(512, 512)) # 方形:放行 + + with pytest.raises(MasterRejected) as e: + gen.generate(card, action, master, _NullProgress(), canvas=(384, 512)) + assert e.value.code is MasterRejectCode.ASPECT_TOO_WIDE diff --git a/backend/tests/test_character_contract.py b/backend/tests/test_character_contract.py index ebb53098..4db299b1 100644 --- a/backend/tests/test_character_contract.py +++ b/backend/tests/test_character_contract.py @@ -4,15 +4,23 @@ 朝向拼错、帧数字段名打错、规格自相矛盾。这些错误以前一路放行到 i2v 调用之后才在画面上显形， 一次误判的成本 = 一次付费视频生成 + 人肉看片。 -本文件只测 DTO 自身,不 import 上层包 —— 契约包要能独立验证。配套的实现侧断言 -("prompt 模板真的按 facing 选对"、"strategy 真的按 n_frames 出帧")随各自的实现 -分片走,契约合法不代表实现读对了。 +契约本身的断言在 feat/character-domain-models 那一片,只测 DTO、不 import 上层包。 +本分片引入 prompt 模块,于是把**实现侧**的配套断言补在这里:类型注解不是运行期约束, +``build_*(facing="sidee")`` 必须当场炸。契约合法不代表实现读对了。 """ from __future__ import annotations import pytest from pydantic import ValidationError +from windup_ai_engine.prompt import ( + WALK_BODY_FRONT, + WALK_BODY_SIDE, + build_attack_prompt, + build_idle_prompt, + build_jump_prompt, + build_walk_prompt, +) from windup_common.models import ( DEFAULT_N_FRAMES, ActionSpec, @@ -90,6 +98,48 @@ def test_unknown_field_name_is_rejected_not_silently_dropped(): CharacterCard(name="n", desc="d", nmae="typo") +# ── A1 实现侧:build_* 是普通函数，注解不构成运行期约束 ────────────────────── + + +@pytest.mark.parametrize( + "build", [build_walk_prompt, build_jump_prompt, build_idle_prompt, build_attack_prompt] +) +def test_prompt_builders_reject_illegal_facing(build): + """直接调 build_*(facing="sidee") 仍要炸。 + + 类型注解不是运行期约束。若把校验删成 ``SIDE if facing == Facing.SIDE else FRONT`` + 的二分，"sidee" 会静默落到 FRONT 模板 —— 正面走的提示词配侧面母版， + 模型靠转身调和矛盾，而调用方什么错都收不到。 + """ + with pytest.raises(ValueError): + build(facing="sidee") + + +@pytest.mark.parametrize( + "build", [build_walk_prompt, build_jump_prompt, build_idle_prompt, build_attack_prompt] +) +def test_prompt_builders_accept_enum_and_legal_string_alike(build): + assert build(facing=Facing.FRONT) == build(facing="front") + assert build(facing=Facing.SIDE) == build(facing="side") + + +def test_walk_prompt_picks_the_template_that_matches_facing(): + """选模板的方向不能反 —— 只验"不炸"验不出模板接反。""" + side = build_walk_prompt(facing=Facing.SIDE) + front = build_walk_prompt(facing=Facing.FRONT) + assert side != front + assert side == WALK_BODY_SIDE.format(garment="the cape and tabard") + assert front == WALK_BODY_FRONT.format(garment="the cape and tabard") + assert "SIDE VIEW facing right" in side and "SIDE VIEW facing right" not in front + assert "FACING THE VIEWER" in front and "FACING THE VIEWER" not in side + + +@pytest.mark.parametrize("build", [build_jump_prompt, build_idle_prompt, build_attack_prompt]) +def test_other_builders_also_switch_body_by_facing(build): + assert build(facing=Facing.SIDE) != build(facing=Facing.FRONT) + assert "FACING THE VIEWER" in build(facing=Facing.FRONT) + + # ── A2 n_frames 是显式字段，不再由 len(poses) 推导 ────────────────────────── diff --git a/backend/tests/test_extract_streaming.py b/backend/tests/test_extract_streaming.py new file mode 100644 index 00000000..7ed1fcbc --- /dev/null +++ b/backend/tests/test_extract_streaming.py @@ -0,0 +1,135 @@ +"""抽帧必须流式,不能把整段视频 materialize 出来(2026-08-10 机器审 P2)。 + +原先 ``iio.imread`` 一次性读出 ``(T, H, W, C)``。实测 121 帧 720p 的真实 i2v 视频, +进程 RSS 峰值 488 MiB,而抽 16 帧只需要其中 16 帧;并发 worker 叠加时这是实打实的内存墙。 +改成 ``imiter`` 后同一段视频 126 MiB(降 74%);抽 8 帧降 79%。 + +如实说明降幅的边界:``extract_all_frames_bytes(cap=150)``(周期检测用)会保留全部 121 帧, +只降 42% —— 省掉的是那个完整 ndarray,保留帧本身该占的内存还在。 +""" +from __future__ import annotations + +import numpy as np +import pytest +from PIL import Image + +from windup_ai_engine.slicing.extract import ( + _extract_frames, + _frame_count, + _uniform_indices, + extract_frames_bytes, +) + + +class _WentThroughImread(BaseException): + """故意继承 BaseException 而不是 Exception —— 见下面用例的 docstring。""" + + +def _forbidden(*a, **k): + raise _WentThroughImread("走了 imread:整段视频被 materialize 了") + + +@pytest.fixture(scope="module") +def video(tmp_path_factory) -> str: + """20 帧的合成视频,每帧一个可辨认的灰度值,用来验"抽到的是哪几帧"。""" + iio = pytest.importorskip("imageio.v3") + path = tmp_path_factory.mktemp("v") / "ramp.mp4" + # 每帧填 i*12,H.264 有损但相邻帧差 12 足以区分;尺寸取 16 的倍数避开编码器 padding。 + frames = [np.full((64, 64, 3), i * 12, dtype=np.uint8) for i in range(20)] + iio.imwrite(path, np.stack(frames), plugin="pyav", codec="libx264") + return str(path) + + +# ── 下标计算 ──────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize(("total", "n", "expect"), [ + (20, 1, [0]), # n=1 不能撞 /(n-1) 除零 + (1, 8, [0]), # 要的比有的多:给全部,不重复 + (1, 1, [0]), + (20, 20, list(range(20))), + (20, 2, [0, 19]), # 必须含首尾 +]) +def test_uniform_indices_covers_the_boundaries(total, n, expect): + assert _uniform_indices(total, n) == expect + + +def test_uniform_indices_never_exceeds_total(): + for total in (1, 3, 20, 121): + for n in (1, 5, 16, 150): + idx = _uniform_indices(total, n) + assert len(idx) == min(n, total) + assert len(set(idx)) == len(idx), "下标不该重复——重复等于同一帧算两帧" + assert idx == sorted(idx) and idx[0] == 0 + # n=1 取首帧(与改造前的实现一致,已在 14 段真实视频上验过逐像素相同); + # 关键姿势的选择归 pick_oneshot,不由抽帧层猜。 + assert len(idx) == 1 or idx[-1] == total - 1 + + +# ── 流式:不许再整段读入 ───────────────────────────────────────────────────── + + +def test_extraction_does_not_materialise_the_whole_video(video, monkeypatch): + """把 ``imread`` 换成炸弹:仍能抽帧,才说明走的是逐帧迭代。 + + 这是本文件的核心断言 —— 改回 ``iio.imread`` 会让它变红(变异测试确认)。 + + 炸弹必须抛 ``BaseException`` 的子类:``_extract_frames`` 用 ``except Exception`` + 兜底到 ffmpeg,抛 ``AssertionError`` 会被它吞掉、静默走 ffmpeg 分支产出正确帧数, + 这条用例于是变成摆设 —— 2026-08-10 变异测试逮到,原版正是这么写的。 + """ + import imageio.v3 as iio + + monkeypatch.setattr(iio, "imread", _forbidden) + frames = _extract_frames(video, 5) + assert len(frames) == 5 + assert all(isinstance(f, Image.Image) and f.mode == "RGBA" for f in frames) + + +def test_extracted_frames_are_the_uniformly_spaced_ones(video): + """抽的是首尾与均匀分布的那几帧,不是前 n 帧。 + + 合成视频每帧灰度递增,所以取出来的灰度序列必须是递增且跨越全程的。 + """ + frames = _extract_frames(video, 5) + greys = [int(np.asarray(f.convert("L")).mean()) for f in frames] + assert greys == sorted(greys), f"帧序错乱:{greys}" + assert greys[0] < 30, f"首帧不是第 0 帧(灰度 {greys[0]})" + assert greys[-1] > 200, f"末帧不是最后一帧(灰度 {greys[-1]})" + + +def test_asking_for_more_frames_than_the_video_has_returns_all(video): + """要 150 帧、视频只有 20 帧:给 20 帧,不静默补帧也不报错。""" + assert len(_extract_frames(video, 150)) == 20 + + +def test_bytes_entry_point_streams_too(video, monkeypatch): + """公开入口是 bytes 版,它也必须走流式(它只是多包了一层临时文件)。""" + import imageio.v3 as iio + + monkeypatch.setattr(iio, "imread", _forbidden) + with open(video, "rb") as f: + assert len(extract_frames_bytes(f.read(), 4)) == 4 + + +# ── 帧数元数据不可信时的兜底 ───────────────────────────────────────────────── + + +def test_frame_count_falls_back_to_counting_when_metadata_is_useless(video, monkeypatch): + """容器元数据在 14 段真实视频上都准,但不同编码的 n_frames 并非都可靠。 + + 元数据给 0 / None / 抛错时必须退回逐帧计数——按错的帧数算下标会抽出错位的帧, + 那是"看起来成功"的失败(帧数对、内容错)。 + """ + import imageio.v3 as iio + + assert _frame_count(video) == 20 + + class _Props: + shape = (0, 64, 64, 3) + + monkeypatch.setattr(iio, "improps", lambda *a, **k: _Props()) + assert _frame_count(video) == 20, "元数据报 0 帧时没退回计数" + + monkeypatch.setattr(iio, "improps", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("no"))) + assert _frame_count(video) == 20, "元数据抛错时没退回计数" diff --git a/backend/tests/test_frame_timing_and_master_prep.py b/backend/tests/test_frame_timing_and_master_prep.py new file mode 100644 index 00000000..257ece24 --- /dev/null +++ b/backend/tests/test_frame_timing_and_master_prep.py @@ -0,0 +1,136 @@ +"""逐帧时长与母版预处理 —— 两者都在每次生成的主路径上，此前无直接覆盖。 + +`frame_durations` 参与每一次出参构造；`prepare_master` 参与每一次 jump / attack 生成。 +纯计算，无需联网。 +""" +from __future__ import annotations + +import io + +import pytest +from PIL import Image + +from windup_ai_engine.postprocess import DEFAULT_FPS_MS, frame_durations +from windup_ai_engine.master_prep import add_headroom, prepare_master + + +# ── frame_durations ────────────────────────────────────────────────────────── +# +# 契约：等时长会让动作发飘、没有重量感，故各动作有不同基准，关键帧还要加长定格。 +# 下面的断言锁的是"动作之间必须有区分度"与"定格必须真的更长"，不是锁具体数值。 + + +def test_durations_length_matches_frame_count(): + assert len(frame_durations("walk", 16)) == 16 + assert frame_durations("walk", 0) == [] + + +def test_each_action_has_its_own_base_duration(): + """idle 慢、run 快 —— 若所有动作退化成同一个值，本用例失败。""" + idle = frame_durations("idle", 4)[0] + walk = frame_durations("walk", 4)[0] + run = frame_durations("run", 4)[0] + assert idle > walk > run, f"idle={idle} walk={walk} run={run} 应递减" + assert idle == DEFAULT_FPS_MS["idle"] + + +def test_unknown_action_falls_back_to_walk_not_zero(): + """未知动作要有可用的兜底，不能返回 0 或抛错——上游动作类型可能先于本模块扩展。""" + assert frame_durations("no_such_action", 3) == frame_durations("walk", 3) + + +def test_key_frame_is_held_longer_than_its_neighbours(): + """攻击触点 / 跳跃顶点要定格，否则动作没有重量感。""" + d = frame_durations("attack", 8, key_frame=3, hold_ms=180) + assert d[3] == 180 + assert d[3] > d[2] and d[3] > d[4] + assert sum(1 for x in d if x == 180) == 1, "只应定格一帧" + + +def test_key_frame_out_of_range_is_ignored_not_crashing(): + """越界的 key_frame 不应炸 —— 帧数由选帧决定，调用方未必对齐。""" + assert frame_durations("attack", 4, key_frame=99) == frame_durations("attack", 4) + assert frame_durations("attack", 4, key_frame=-1) == frame_durations("attack", 4) + + +def test_hold_never_shortens_a_frame(): + """hold_ms 小于基准时长时取基准，定格不能反而变快。""" + base = DEFAULT_FPS_MS["idle"] # 450，远大于常见 hold 180 + d = frame_durations("idle", 4, key_frame=1, hold_ms=100) + assert d[1] == base + + +# ── prepare_master ─────────────────────────────────────────────────────────── +# +# 契约：jump 向上腾空、attack 挥砍过头顶，都会顶出视频画面上沿（实测 attack 15/72 帧触顶）。 +# 故这两个动作要在母版顶部补空间，其余动作原样返回。 + + +def _png(w: int, h: int, fill=(200, 60, 60), bg=(18, 220, 30)) -> bytes: + """一张四角为纯背景色、中下部有主体的图。背景取绿幕色以便断言补边颜色。""" + img = Image.new("RGB", (w, h), bg) + for y in range(h // 3, h): + for x in range(w // 3, w * 2 // 3): + img.putpixel((x, y), fill) + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + +def _size(b: bytes) -> tuple[int, int]: + return Image.open(io.BytesIO(b)).size + + +@pytest.mark.parametrize("action", ["jump", "attack"]) +def test_airborne_actions_get_headroom(action: str): + src = _png(64, 100) + out = prepare_master(src, action) + w0, h0 = _size(src) + w1, h1 = _size(out) + assert w1 == w0, "宽度不应变化" + assert h1 > h0, f"{action} 必须补顶部空间，否则腾空时头顶顶出画面被裁" + + +@pytest.mark.parametrize("action", ["walk", "run", "idle", "hit", "unknown"]) +def test_other_actions_are_returned_untouched(action: str): + """不需要处理的动作必须**原样**返回 —— 无谓的重编码会引入压缩损失。""" + src = _png(64, 100) + assert prepare_master(src, action) is src + + +def test_headroom_is_added_on_top_and_original_sits_at_bottom(): + """补的边必须在**顶部**：原图贴底，顶部新增区域应为背景色。""" + src = _png(64, 90) + out = add_headroom(src, ratio=0.6) + src_img = Image.open(io.BytesIO(src)).convert("RGB") + out_img = Image.open(io.BytesIO(out)).convert("RGB") + added = out_img.height - src_img.height + assert added > 0 + + # 顶部新增区域 = 背景色 + assert out_img.getpixel((2, 2)) == src_img.getpixel((0, 0)) + # 原图整体落在底部：最后一行应与原图最后一行一致 + assert out_img.crop((0, out_img.height - 1, out_img.width, out_img.height)).tobytes() == \ + src_img.crop((0, src_img.height - 1, src_img.width, src_img.height)).tobytes() + + +def test_smaller_ratio_gives_more_headroom(): + """ratio 是"角色占画面高度的比例"，越小顶部留白越多。""" + src = _png(64, 100) + _, h_loose = _size(add_headroom(src, ratio=0.5)) + _, h_tight = _size(add_headroom(src, ratio=0.9)) + assert h_loose > h_tight + + +def test_jump_gets_more_headroom_than_attack(): + """jump 向上腾空，需要的顶部空间比 attack 的过顶挥砍更多（0.62 vs 0.70）。""" + src = _png(64, 100) + _, h_jump = _size(prepare_master(src, "jump")) + _, h_attack = _size(prepare_master(src, "attack")) + assert h_jump > h_attack + + +@pytest.mark.parametrize("bad", [0.0, 0.1, 1.0, 1.5, -0.3]) +def test_invalid_ratio_raises(bad: float): + with pytest.raises(ValueError, match="ratio"): + add_headroom(_png(32, 32), ratio=bad) diff --git a/backend/tests/test_generation_orchestration.py b/backend/tests/test_generation_orchestration.py new file mode 100644 index 00000000..5ae0f6d5 --- /dev/null +++ b/backend/tests/test_generation_orchestration.py @@ -0,0 +1,285 @@ +"""生成任务编排端到端(离线):提交任务 → 后台调 ai_engine 出帧 → 上传 → 写回结果。 + +用内存 sqlite + 真实 GenerationTaskRecord ORM + 真实 AiGenerationService + 真实 +CharacterGenerator(视频 provider / matte / 抽帧全部桩替,不联网、不碰对象存储)。 +证明"任务 → ai_engine → 帧 → COMPLETED"这条链真能跑通。 +""" +from __future__ import annotations + +import io + +import pytest +from PIL import Image +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from windup_framework.db.base import Base +from windup_app.server.project.model import Project # 注册 windup_project 表(create_all 用) +from windup_app.server.orchestrator.model import ( + ActionType, + CharacterActionInput, + CharacterActionOutput, + TaskStatus, +) +from windup_app.server.orchestrator.executor import ActionTaskExecutor +from windup_app.server.orchestrator.service import AiGenerationService +from windup_ai_engine.impl import CharacterGenerator +from windup_ai_engine.strategy.concrete import VideoFrameStrategy +from windup_common.models import GenRoute + + +def _tiny_png(shift: int = 0) -> bytes: + img = Image.new("RGBA", (64, 96), (0, 0, 0, 0)) + for y in range(20, 80): + for x in range(24 + shift, 40 + shift): + img.putpixel((x, y), (200, 60, 60, 255)) + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + +class _StubVideo: + def i2v(self, first_frame, prompt, seconds=5, size="1280x720"): + return b"fake-mp4" + + +class _StubMatte: + def cutout(self, frame): + return frame + + +@pytest.fixture +def session_factory(): + """共享的内存 sqlite(StaticPool 保证多 session 同库),建好任务表。""" + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + return sessionmaker(bind=engine) + + +def _real_offline_generator(monkeypatch) -> CharacterGenerator: + """真实 CharacterGenerator,但抽帧顶替成合成帧(不解码 mp4 / 不联网)。""" + dense = [ + Image.open(io.BytesIO(_tiny_png(shift=i % 6))).convert("RGBA") + for i in range(24) + ] + monkeypatch.setattr( + "windup_ai_engine.strategy.concrete.extract_all_frames_bytes", + lambda video, cap=150: dense, + ) + return CharacterGenerator( + {GenRoute.VIDEO_I2V: VideoFrameStrategy(_StubVideo(), _StubMatte())} + ) + + +def test_action_task_runs_end_to_end(session_factory, monkeypatch): + uploaded: list[bytes] = [] + + def _upload(png: bytes) -> str: + uploaded.append(png) + return f"https://cdn.example.com/frame-{len(uploaded)}.png" + + service = AiGenerationService() + executor = ActionTaskExecutor( + generator=_real_offline_generator(monkeypatch), + upload=_upload, + fetch_master=lambda _input: _tiny_png(), + session_factory=session_factory, + ) + action_input = CharacterActionInput( + character_id=1, action_type=ActionType.WALK, num_frames=6, + ) + + # 1) 提交:建 PENDING 任务 + with session_factory() as s: + task = service.generate_character_action(s, user_id=1, input=action_input) + s.commit() + task_id = task.id + assert task.status is TaskStatus.PENDING + + # 2) 后台跑(自开 session) + executor.run_action_task(task_id, action_input) + + # 3) 轮询:任务 COMPLETED,结果是含 URL 的帧序列 + with session_factory() as s: + done = service.get_task(s, project_id=1, task_id=task_id) + assert done is not None + assert done.status is TaskStatus.COMPLETED + assert isinstance(done.result, CharacterActionOutput) + assert done.result.action_type == "walk" + assert len(done.result.frames) >= 1 + assert uploaded, "应逐帧上传" + for i, frame in enumerate(done.result.frames): + assert frame.index == i + assert frame.image_url.startswith("https://") + assert frame.duration_ms is not None + + +def _png_of(w: int, h: int) -> bytes: + """指定尺寸的一张带主体的 PNG。""" + img = Image.new("RGBA", (w, h), (0, 0, 0, 0)) + img.paste((200, 60, 60, 255), (w // 4, h // 4, w // 2, h // 2)) + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + +class _SpyGenerator: + """记录传入的 facing / canvas,验证项目约束确实喂进了 ai_engine。 + + ``canvas`` 是必须接的:交付尺寸现在由引擎按项目 sprite 尺寸出帧负责,编排层 + 不再拿到帧之后自己缩 —— 那一步用 thumbnail 补边,只缩不放,会把引擎对齐好的 + 脚线挪走。本 spy 照真实引擎的约定按 canvas 出帧。 + + ``honour_canvas=False`` 用来模拟"引擎没按尺寸出帧",验证编排层会报错而不是 + 静默缩放补救。 + """ + + def __init__(self, honour_canvas: bool = True) -> None: + self.seen_facing: str | None = None + self.seen_canvas: tuple[int, int] | None = None + self._honour = honour_canvas + + def generate(self, card, action, master, progress, canvas=None): + from windup_ai_engine.ports import GeneratedAction + + self.seen_facing = action.facing + self.seen_canvas = canvas + size = canvas if (canvas and self._honour) else (256, 256) + # 不传 fps:GeneratedAction 早已删掉该字段(播放时序的唯一真相源是 durations)。 + # 这个 spy 之前一直在传,构造直接 TypeError、任务被判 FAILED —— 而当时的用例 + # 只断言 seen_facing(在构造之前就赋了值),于是**用例绿着、任务其实是失败的**。 + from windup_ai_engine.ports import ActionQuality + + return GeneratedAction( + frames=[_png_of(*size)], + durations=[100], + quality=ActionQuality(motion_scale=1.0, dead_frames=[], loop_seam=None), + ) + + +def test_project_perspective_constrains_facing(session_factory): + # perspective=2 → front(见 executor._PERSPECTIVE_TO_FACING) + with session_factory() as s: + proj = Project( + user_id=1, project_name="p", character_perspective=2, + directional_movement=1, sprite_width=64, sprite_height=64, + ) + s.add(proj) + s.commit() + project_id = proj.id + + spy = _SpyGenerator() + executor = ActionTaskExecutor( + generator=spy, + upload=lambda _png: "https://cdn.example.com/f.png", + fetch_master=lambda _input: _tiny_png(), + session_factory=session_factory, + ) + action_input = CharacterActionInput( + character_id=1, action_type=ActionType.WALK, num_frames=2, + ) + with session_factory() as s: + task = AiGenerationService().generate_character_action(s, user_id=1, input=action_input) + s.commit() + task_id = task.id + + executor.run_action_task(task_id, action_input, project_id) # 带项目约束 + + assert spy.seen_facing == "front", "项目 perspective 应约束生成朝向" + + +def test_action_task_marks_failed_on_error(session_factory): + def _boom(_input): + raise RuntimeError("母版下载失败") + + service = AiGenerationService() + executor = ActionTaskExecutor( + generator=None, # 不会用到:取母版先炸 + fetch_master=_boom, + session_factory=session_factory, + ) + action_input = CharacterActionInput( + character_id=1, action_type=ActionType.WALK, num_frames=4, + ) + with session_factory() as s: + task = service.generate_character_action(s, user_id=1, input=action_input) + s.commit() + task_id = task.id + + executor.run_action_task(task_id, action_input) # 不抛,兜底为 FAILED + + with session_factory() as s: + done = service.get_task(s, project_id=1, task_id=task_id) + assert done.status is TaskStatus.FAILED + assert "母版下载失败" in (done.error_message or "") + + +# ── 交付尺寸传给引擎(2026-08-11 挣得)────────────────────────────────────────── +# +# 这里以前是拿到 256 的帧再 _fit_to 到项目 sprite 尺寸。那步用 Image.thumbnail 补边, +# 而 thumbnail **只缩不放**:项目要 512 时帧根本不会被放大,只是原尺寸居中贴进 512 +# 画布,于是引擎刚对齐好的脚线 0.92 被挪到 0.709(实测),角色不站在地上。 +# 现在尺寸交给引擎(canvas),编排层只核对、不缩放。 + + +def _run_with_project(session_factory, spy, sprite=(64, 64)): + """建一个指定 sprite 尺寸的项目,跑一次动作任务,返回 (task_id, project_id)。""" + with session_factory() as s: + proj = Project( + user_id=1, project_name="p", character_perspective=1, + directional_movement=1, sprite_width=sprite[0], sprite_height=sprite[1], + ) + s.add(proj) + s.commit() + project_id = proj.id + + executor = ActionTaskExecutor( + generator=spy, + upload=lambda _png: "https://cdn.example.com/f.png", + fetch_master=lambda _input: _tiny_png(), + session_factory=session_factory, + ) + action_input = CharacterActionInput( + character_id=1, action_type=ActionType.WALK, num_frames=2, + ) + with session_factory() as s: + task = AiGenerationService().generate_character_action( + s, user_id=1, input=action_input + ) + s.commit() + task_id = task.id + executor.run_action_task(task_id, action_input, project_id) + return task_id, project_id + + +def test_project_sprite_size_is_passed_to_the_engine(session_factory): + """项目 sprite 尺寸必须作为 canvas 传进引擎 —— 而不是拿到帧再缩。""" + spy = _SpyGenerator() + _run_with_project(session_factory, spy, sprite=(512, 512)) + assert spy.seen_canvas == (512, 512), "引擎应当收到项目 sprite 尺寸" + + +def test_non_square_project_sprite_size_is_passed_through(session_factory): + """非方形项目尺寸也要原样传下去,不能只传一个边长。""" + spy = _SpyGenerator() + _run_with_project(session_factory, spy, sprite=(384, 512)) + assert spy.seen_canvas == (384, 512) + + +def test_engine_frame_of_wrong_size_fails_instead_of_being_rescaled(session_factory): + """引擎没按尺寸出帧 → 任务失败,**不做静默缩放补救**。 + + 以前这里会 _fit_to 补救,把"引擎没按尺寸出帧"抹平,代价是脚线对齐被破坏 —— + 正是本仓最忌讳的"看起来成功的错产物"。 + """ + spy = _SpyGenerator(honour_canvas=False) # 恒出 256,无视 canvas + task_id, _ = _run_with_project(session_factory, spy, sprite=(512, 512)) + with session_factory() as s: + done = AiGenerationService().get_task(s, project_id=1, task_id=task_id) + assert done.status is TaskStatus.FAILED, "尺寸对不上必须失败,不能悄悄缩放交付" + assert "512" in (done.error_message or ""), "报错要说清期望尺寸" diff --git a/backend/tests/test_generation_stream_auth.py b/backend/tests/test_generation_stream_auth.py new file mode 100644 index 00000000..e5f7db11 --- /dev/null +++ b/backend/tests/test_generation_stream_auth.py @@ -0,0 +1,246 @@ +"""SSE 订阅的归属校验与终态预检。 + +两条都是"测试全绿的情况下"存在的缺口： +- 主线 #110 已经校验了"项目属于当前用户"，但**没有**校验"任务属于那个项目"。缺这一道， + 任意已认证用户拿自己的 project_id 配上别人的 task_id 就能订阅到别人的流，而事件体带 + result，即最终帧的对象存储 URL。 +- 终态预检原先是一行 TODO，而端点 docstring 已经承诺了该行为 —— 读文档的人不会发现， + 实际表现是客户端要先挂满一次心跳超时才拿到终态。 + +归属口径沿用主线：靠 project_id 而不是任务自己的 user_id。我此前那版删掉了 project_id +改用 task.user_id，方向是错的 —— project_id 在主线里正是归属校验的依据，且 EventBus +按 (project_id, task_id) 双键隔离，删掉它会退化主线已有的能力。 +""" +from __future__ import annotations + +import json + +import pytest + +from windup_app.server.orchestrator import task_repo +from windup_app.server.orchestrator.model import GenerationType, TaskStatus + +# SSE 事件体的键集是**对外契约**，故在这里写死。 +# 不要用 task_event_payload(task) 反算期望值 —— 那样两边同源，删字段时一起变、断言永远 +# 成立（2026-08-11 变异测试逮到第一版正是如此：删掉 error_message 仍全绿）。 +_EVENT_KEYS = { + "id", "user_id", "project_id", "task_type", + "status", "input_payload", "result", "error_message", +} + + +def _create_project(client, name: str = "SSE 项目") -> dict: + return client.post("/projects", json={ + "project_name": name, + "character_perspective": 1, + "directional_movement": 2, + "sprite_width": 64, + "sprite_height": 64, + }).json()["data"] + + +@pytest.fixture() +def session(engine): + """绑定测试 engine 的 session，并补建 generation_task 表。 + + conftest 的 ``engine`` fixture 只建了 project / user / character / workflow_run + 四张，本 PR 新引入的这张不在那份清单里。在这里补建而不是改公共 fixture， + 是为了不影响其它测试文件的建表集合。 + """ + from sqlalchemy.orm import sessionmaker + + from windup_app.server.orchestrator.model import GenerationTaskRecord + from windup_framework.db import Base + + Base.metadata.create_all(engine, tables=[GenerationTaskRecord.__table__]) + s = sessionmaker(bind=engine, expire_on_commit=False)() + yield s + s.close() + + +def _make_task(session, *, user_id: int, project_id: int, + status: TaskStatus = TaskStatus.PENDING) -> int: + """直接落一条任务，绕过端点（端点会真的起后台线程去跑生成）。""" + task = task_repo.create_task( + session, + user_id=user_id, + project_id=project_id, + task_type=GenerationType.CHARACTER_IMAGE, + input_payload={"prompt": "x"}, + ) + if status is not TaskStatus.PENDING: + task_repo.update_status(session, task.id, status) + session.commit() + return task.id + + +# ── ① 任务必须属于所声明的项目 ────────────────────────────────────────────── + + +def test_task_from_another_project_cannot_be_subscribed(auth_client, session): + """主线只校验了"项目属于我"，没校验"任务属于该项目"。 + + 缺这一道：**用自己的项目 id 配别人的任务 id** 就能订阅到别人的流。本用例用同一个 + 用户的两个项目复现，因此排除了"项目归属校验挡住了"这种解释 —— 两个项目都属于我， + 唯一的区别是任务不在我声明的那个项目里。 + """ + mine = _create_project(auth_client, "我的项目") + other = _create_project(auth_client, "另一个项目") + task_id = _make_task(session, user_id=1, project_id=other["id"], + status=TaskStatus.COMPLETED) + + with auth_client.stream( + "GET", f"/generation/tasks/{task_id}/stream", + params={"project_id": mine["id"]}, + ) as r: + body = r.read().decode() + assert "event: " not in body, f"跨项目订阅拿到了事件流：{body[:200]}" + + +def test_task_in_the_declared_project_is_subscribable(auth_client, session): + """对照组：任务确实在所声明的项目里时必须放行 —— 否则上一条可能只是全都拒了。""" + project = _create_project(auth_client) + task_id = _make_task(session, user_id=1, project_id=project["id"], + status=TaskStatus.COMPLETED) + + with auth_client.stream( + "GET", f"/generation/tasks/{task_id}/stream", + params={"project_id": project["id"]}, + ) as r: + body = r.read().decode() + assert "event: completed" in body, body[:200] + + +def test_rejection_happens_before_subscribing(auth_client, session): + """校验必须在 subscribe **之前**。 + + 放在之后的话，越权请求仍会在 EventBus 上挂一个订阅者 —— 它照样收到事件，只是响应体 + 被丢弃；订阅表还会因为没人 unsubscribe 而增长。 + """ + from windup_app.web.api.generation import event_bus + + mine = _create_project(auth_client, "我的项目") + other = _create_project(auth_client, "另一个项目") + task_id = _make_task(session, user_id=1, project_id=other["id"]) + + with auth_client.stream( + "GET", f"/generation/tasks/{task_id}/stream", + params={"project_id": mine["id"]}, + ) as r: + r.read() + key = (mine["id"], task_id) + assert not event_bus._queues.get(key), "越权请求在 EventBus 上留下了订阅者" + + +# ── ② 终态预检 ───────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize(("status", "expected"), [ + (TaskStatus.COMPLETED, "event: completed"), + (TaskStatus.FAILED, "event: failed"), +]) +def test_already_terminal_task_gets_its_event_immediately(auth_client, session, status, expected): + """订阅时任务已终结 → 立即推终态并关闭，而不是先挂满一次心跳超时。""" + project = _create_project(auth_client) + task_id = _make_task(session, user_id=1, project_id=project["id"], status=status) + + with auth_client.stream( + "GET", f"/generation/tasks/{task_id}/stream", + params={"project_id": project["id"]}, + ) as r: + body = r.read().decode() + assert expected in body, body[:300] + assert "heartbeat" not in body, "先发了心跳 = 没有走终态预检" + + +def test_terminal_event_body_matches_the_documented_contract(auth_client, session): + """订阅时补发的终态事件，键集必须与契约一致。""" + project = _create_project(auth_client) + task_id = _make_task(session, user_id=1, project_id=project["id"], + status=TaskStatus.COMPLETED) + + with auth_client.stream( + "GET", f"/generation/tasks/{task_id}/stream", + params={"project_id": project["id"]}, + ) as r: + body = r.read().decode() + line = next(x for x in body.splitlines() if x.startswith("data: ")) + assert set(json.loads(line[6:])) == _EVENT_KEYS + + +def test_both_send_paths_use_the_same_payload_builder(session): + """运行中推送与终态补发必须同形状 —— 两处各抄一份字段列表迟早分叉。 + + 直接比两条真实路径的产出：``_publish_task_update``（运行中）与 + ``task_event_payload``（终态预检用的那个）。 + """ + task_id = _make_task(session, user_id=1, project_id=42, status=TaskStatus.COMPLETED) + task = task_repo.get_task(session, task_id) + + sent: list[dict] = [] + + class _Bus: + def publish(self, project_id, tid, event, data): + sent.append(data) + + old_bus = task_repo._event_bus + task_repo._event_bus = _Bus() + try: + task_repo._publish_task_update(task_id, task) + finally: + task_repo._event_bus = old_bus + + assert set(sent[0]) == _EVENT_KEYS + assert set(task_repo.task_event_payload(task)) == _EVENT_KEYS + + +@pytest.mark.parametrize("status", [TaskStatus.PENDING, TaskStatus.RUNNING]) +def test_non_terminal_status_is_not_mistaken_for_terminal(status): + """非终态不能被预检判成终态，否则连接刚建立就被关掉。 + + **本条不走 HTTP，如实说明原因**：非终态的流是无限心跳，靠 + ``request.is_disconnected()`` 退出，而 TestClient 下它不会变 True —— 生成器永不 + 结束，TestClient 在 teardown 上阻塞（第一版这么写，把 pytest 挂满 10 分钟）。 + 压低心跳间隔也无效，因为挂的不是等待、是退出条件。 + + 所以这里直接测预检用的那个判据函数。它是终态预检的唯一入口，改坏了上面那几条终态 + 用例会红，因此覆盖不算空缺。 + """ + from windup_app.server.orchestrator.model import GenerationTask + + task = GenerationTask(id=1, user_id=1, project_id=42, + task_type=GenerationType.CHARACTER_IMAGE, status=status) + assert task_repo.terminal_event_for(task) is None + + +# ── ③ project_id 为空的任务发不出事件，要记 warning 而不是静默丢 ────────────── + + +def test_task_without_project_id_logs_instead_of_publishing_into_the_void(session, caplog): + """EventBus 按 (project_id, task_id) 索引，project_id 为空就发不到任何订阅者。 + + 静默 publish 出去的话，现象是"任务确实在跑、状态也在落库，但前端进度条一动不动"， + 而日志里一行异常都没有 —— 属于最难查的那类静默失败。 + """ + import logging + + task_id = _make_task(session, user_id=1, project_id=42) + task = task_repo.get_task(session, task_id) + task.project_id = None + + sent: list = [] + + class _Bus: + def publish(self, *a): + sent.append(a) + + old_bus = task_repo._event_bus + task_repo._event_bus = _Bus() + try: + with caplog.at_level(logging.WARNING): + task_repo._publish_task_update(task_id, task) + finally: + task_repo._event_bus = old_bus + + assert sent == [], "不该发到一个没人听的键上" + assert any("project_id" in r.message for r in caplog.records), "应记 warning" diff --git a/backend/tests/test_loop.py b/backend/tests/test_loop.py new file mode 100644 index 00000000..daac0c1e --- /dev/null +++ b/backend/tests/test_loop.py @@ -0,0 +1,94 @@ +"""循环闭合(周期检测 + 单周期取帧)测试 —— 纯 CV,无需联网。""" + +import pytest +from PIL import Image + +from windup_ai_engine.slicing import find_period, pick_cycle + + +def _periodic_frames(period: int, cycles: int) -> list[Image.Image]: + """构造已知周期的帧序列:亮度按周期正弦变化(每帧一张纯灰图)。""" + import math + + frames = [] + for i in range(period * cycles): + v = int(128 + 100 * math.sin(2 * math.pi * i / period)) + frames.append(Image.new("RGB", (48, 48), (v, v, v))) + return frames + + +def test_find_period_detects_known_period(): + frames = _periodic_frames(period=20, cycles=5) + p = find_period(frames) + assert abs(p - 20) <= 1 # 检出周期 ≈ 真值 + + +def test_pick_cycle_returns_n_frames(): + frames = _periodic_frames(period=20, cycles=5) + out = pick_cycle(frames, 8) + assert len(out) == 8 + + +def _ramp_frames(n: int = 40) -> list[Image.Image]: + """亮度单调上升 = 无周期,专走"测不到周期"那条分支。""" + return [Image.new("RGB", (48, 48), (i * 4,) * 3) for i in range(n)] + + +def test_pick_cycle_rejects_insufficient_source(): + """源帧不够就报错(原本原样返回 4 帧,冒充"抽到了 8 帧的循环")。""" + frames = _periodic_frames(period=4, cycles=1) # 4 帧 < 8 + with pytest.raises(ValueError, match=r"源帧不足.*8.*4"): + pick_cycle(frames, 8) + with pytest.raises(ValueError, match="源帧不足"): # 只差一帧也不放过(挡 off-by-one) + pick_cycle(frames, len(frames) + 1) + + +def test_pick_cycle_passthrough_when_n_equals_len(): + frames = _periodic_frames(period=4, cycles=2) # 8 帧 == 8 + assert pick_cycle(frames, 8) is frames + + +def test_pick_cycle_rejects_non_positive_n(): + """n<=0 两条分支的旧行为都不可接受:检出周期时 IndexError,测不到周期时静默返回 []。""" + for frames in (_periodic_frames(period=20, cycles=5), _ramp_frames()): + for n in (0, -1): + with pytest.raises(ValueError, match="n 必须"): + pick_cycle(frames, n) + + +def test_pick_cycle_n1_takes_dominant_pose(): + """n=1 的"循环"只是一张静止姿势:取全片最具代表性的一帧(停留最久的相位), + 而不是首帧 —— i2v 首帧是母版静立姿,单看读不出这是什么动作。""" + a = Image.new("RGB", (48, 48), (200, 200, 200)) + b = Image.new("RGB", (48, 48), (40, 40, 40)) + frames = [b] + [a] * 20 + [b] * 8 + [a] * 11 # a 占多数,首帧刻意放 b + out = pick_cycle(frames, 1) + assert len(out) == 1 + assert out[0] is a + assert out[0] is not frames[0] + + +def test_pick_cycle_returns_exactly_n_distinct_frames_for_every_legal_n(): + """全量扫 n=1..len(frames):两条分支(检出周期 / 测不到周期)都要恒好 n 帧,且互不重复。 + + 长度对但夹着重复帧同样是"看起来成功"的错结果 —— 取样相位塌在一起,循环里会卡一下。 + """ + for frames in (_periodic_frames(period=8, cycles=5), _ramp_frames()): + for n in range(1, len(frames) + 1): + out = pick_cycle(frames, n) + assert len(out) == n, n + pos = [next(i for i, f in enumerate(frames) if f is o) for o in out] + assert len(set(pos)) == n, (n, pos) + + +def test_pick_cycle_closes_the_loop(): + # 取出的一周期,末帧的下一拍应接近首帧(亮度差小) + import numpy as np + + frames = _periodic_frames(period=20, cycles=5) + out = pick_cycle(frames, 8) + first = np.asarray(out[0].convert("L"), float) + last = np.asarray(out[-1].convert("L"), float) + step = np.abs(np.asarray(out[1].convert("L"), float) - first).mean() + seam = np.abs(last - first).mean() + assert seam <= step * 2 + 5 # 回接缝不显著大于一个正常步幅 diff --git a/backend/tests/test_master_check_and_quality.py b/backend/tests/test_master_check_and_quality.py new file mode 100644 index 00000000..cc3bfd43 --- /dev/null +++ b/backend/tests/test_master_check_and_quality.py @@ -0,0 +1,236 @@ +"""母版入口预检 + 出参成色信号。 + +两头各一道闸，方向相反：进门那道在**花钱之前**挡住不可能生成好的输入； +出门那道在钱已花完之后，让上层看得出"这次生成得怎么样"。 + +2026-08-07 的教训：喂一张"人物在画板前作画"的图请求 walk，全程无一处报错， +16 帧构图完整的错角色出完、钱花完。而一段每帧都一样的 walk 与一段步态干净的 walk， +帧数 / 时长 / fps 完全相同，调用方分辨不出。 +""" +from __future__ import annotations + +import io + +import pytest +from PIL import Image + +from windup_ai_engine.master_check import ( + MIN_SUBJECT_SIDE, + REJECT_ASPECT, + check_master, + reject_aspect_for, +) +from windup_ai_engine.ports import ActionQuality, MasterRejectCode, MasterRejected +from windup_ai_engine.slicing import dead_frame_indices, loop_seam, motion_scale +from windup_ai_engine.postprocess.pack import FILL_H, FILL_W + + +def _png(w: int, h: int, blob: tuple[tuple[int, int, int, int], tuple] | None = None, + bg=(0, 0, 0, 0)) -> bytes: + img = Image.new("RGBA", (w, h), bg) + if blob: + (x0, y0, x1, y1), color = blob + for y in range(y0, y1): + for x in range(x0, x1): + img.putpixel((x, y), color) + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + +# ── 入口预检：四种拒绝码 ────────────────────────────────────────────────────── + + +def test_undecodable_bytes_rejected_before_spending(): + """坏 bytes 直接炸，不要等 i2v 花完钱才发现输入根本不是图。""" + with pytest.raises(MasterRejected) as e: + check_master(b"not an image at all") + assert e.value.code is MasterRejectCode.UNDECODABLE + + +def test_fully_transparent_has_no_subject(): + with pytest.raises(MasterRejected) as e: + check_master(_png(200, 200)) + assert e.value.code is MasterRejectCode.NO_SUBJECT + + +def test_flat_single_color_has_no_subject(): + """全同色 = 没有可动的东西。不透明但一片死板的图同样该拒。""" + with pytest.raises(MasterRejected) as e: + check_master(_png(200, 200, bg=(120, 90, 60, 255))) + assert e.value.code is MasterRejectCode.NO_SUBJECT + + +def test_subject_smaller_than_min_side_rejected(): + """包围盒最短边不足 → 下游会把它 NEAREST 放大 20 倍，那是色块不是角色。 + + 刻意用**细长条**而不是小方块：细长条的像素占比高达 1.3%（远超 0.1% 下限）， + 所以占比那条拦不住它，只有最短边这条能拦。用小方块的话两条判据都会触发， + 删掉任何一条测试都照样绿——那种测试等于没写（2026-08-09 变异测试逮到）。 + """ + thin = MIN_SUBJECT_SIDE - 2 # 6px 宽 + with pytest.raises(MasterRejected) as e: + check_master(_png(300, 300, blob=((100, 40, 100 + thin, 240), (200, 60, 60, 255)))) + assert e.value.code is MasterRejectCode.SUBJECT_TOO_SMALL + + +def test_scattered_specks_pass_side_check_but_fail_area_ratio(): + """对角两粒噪点会把包围盒撑到整幅——边长检查全过，占比才拦得住。 + + 这两条判的不是同一件事，缺了占比这条，一张几乎空白的图会被判成"有主体"。 + """ + # 每粒 10×10=100px（边长过得了 MIN_SUBJECT_SIDE=8），两粒共 200px， + # 占 600×600 的 0.056%，压在 0.1% 下限之下；而包围盒被撑到 ~590×590，边长检查全过。 + img = Image.new("RGBA", (600, 600), (0, 0, 0, 0)) + for (x, y) in ((8, 8), (582, 582)): + for dy in range(10): + for dx in range(10): + img.putpixel((x + dx, y + dy), (200, 60, 60, 255)) + buf = io.BytesIO() + img.save(buf, "PNG") + with pytest.raises(MasterRejected) as e: + check_master(buf.getvalue()) + assert e.value.code is MasterRejectCode.SUBJECT_TOO_SMALL + + +def test_extremely_wide_subject_rejected(): + """主体太扁 → 方形画布只能把角色硬缩成一条，不如在花钱前退回去。""" + w = int(60 * REJECT_ASPECT) + 40 + with pytest.raises(MasterRejected) as e: + check_master(_png(w + 40, 200, blob=((10, 60, 10 + w, 120), (200, 60, 60, 255)))) + assert e.value.code is MasterRejectCode.ASPECT_TOO_WIDE + + +def test_ordinary_humanoid_master_passes_and_reports_facts(): + """人形母版必须放行——预检的价值在于不误伤，误伤一次比漏放一次更贵。""" + facts = check_master(_png(400, 600, blob=((160, 100, 240, 520), (200, 60, 60, 255)))) + assert facts.size == (400, 600) + assert 0.1 < facts.subject_ratio < 1.2 + assert facts.subject_area_ratio > 0.001 + assert facts.note() # 进度文案不能是空串 + + +def test_reject_aspect_is_derived_from_canvas_geometry_not_hardcoded(): + """阈值必须跟着画布几何走。把 pack.py 的 FILL_W/FILL_H 改了而这里不动， + 预检就会放行一批下游装不下的母版——那正是"看起来成功"的来源。""" + assert REJECT_ASPECT == pytest.approx(2 * FILL_W / FILL_H) + + +# ── 非方形交付画布下的比例上限(2026-08-11 挣得)──────────────────────────── +# +# REJECT_ASPECT 的推导默认画布是方形(FILL_W / FILL_H 是同一条边长的两个比例)。 +# 交付画布可以非方之后前提不再成立:同一条推导做下来是 REJECT_ASPECT*(cw/ch)。 +# 不跟着收的后果是预检按方形判、出帧按非方出 —— 一个刚好过检的主体在 384×512 +# 画布上交付占高只有 0.2324,而这条阈值本意保证的下限是 FILL_H/2=0.31(实测)。 + + +def test_reject_aspect_for_square_canvas_is_unchanged(): + """方形画布(以及不指定)必须与原来完全一致 —— 默认行为不变。""" + assert reject_aspect_for(None) == REJECT_ASPECT + for c in (128, 256, 512, 1024): + assert abs(reject_aspect_for((c, c)) - REJECT_ASPECT) < 1e-12 + + +def test_reject_aspect_for_narrow_canvas_tightens_proportionally(): + """窄高画布容得下的主体更窄,阈值按 cw/ch 收紧;宽扁画布反之放宽。""" + assert reject_aspect_for((384, 512)) < REJECT_ASPECT + assert reject_aspect_for((512, 384)) > REJECT_ASPECT + assert abs(reject_aspect_for((384, 512)) - REJECT_ASPECT * 384 / 512) < 1e-12 + + +def test_threshold_delivers_exactly_half_target_height_on_any_canvas(): + """**预检几何与出帧几何是同一套**的直接证据。 + + 阈值的定义就是交付主体高退化到目标高度的一半。拿真实出帧验证:处在各自比例 + 上限的主体,在任何形状的画布上交付占高都必须落在 FILL_H/2 附近(差的是取整)。 + """ + import numpy as np + + from windup_ai_engine.postprocess.pack import align_bottom_center + + for cw, ch in ((256, 256), (512, 512), (384, 512), (512, 384), (128, 192)): + limit = reject_aspect_for((cw, ch)) + base_h, src_w = 200, 3000 # 源画幅给足,别让主体被源边界裁掉 + blob_w = int(base_h * limit) + img = Image.new("RGBA", (src_w, 600), (0, 0, 0, 0)) + img.paste((200, 60, 60, 255), (100, 100, 100 + blob_w, 100 + base_h)) + out = align_bottom_center([img], cell=cw, cell_h=ch, ref_height=float(base_h)) + ys, _ = np.nonzero(np.asarray(out[0])[:, :, 3] > 128) + ratio = (int(ys.max()) - int(ys.min()) + 1) / ch + assert abs(ratio - FILL_H / 2) < 0.01, ( + f"{cw}×{ch}: 阈值处交付占高 {ratio:.4f},应为 {FILL_H / 2}" + ) + + +def test_check_master_uses_the_canvas_it_is_given(): + """同一张母版:方形画布放行,窄高画布上超限 → 必须被拒。""" + ratio = (REJECT_ASPECT + reject_aspect_for((384, 512))) / 2 # 夹在两个阈值中间 + bw = int(60 * ratio) + png = _png(bw + 80, 200, blob=((10, 60, 10 + bw, 120), (200, 60, 60, 255))) + + check_master(png, canvas=(512, 512)) # 方形:放行 + with pytest.raises(MasterRejected) as e: + check_master(png, canvas=(384, 512)) # 窄高:同一张图装不下 + assert e.value.code is MasterRejectCode.ASPECT_TOO_WIDE + + +def test_rejection_carries_machine_readable_code_not_just_a_message(): + """server 要据此选文案 / 决定 4xx-不重试，用消息做分支会在改文案时悄悄失效。""" + with pytest.raises(MasterRejected) as e: + check_master(b"broken") + assert isinstance(e.value.code, MasterRejectCode) + assert e.value.detail + + +# ── 出参成色：三个字段各自不可由其他两个推导 ────────────────────────────────── + + +def _frames(n: int, shift: int = 3) -> list[Image.Image]: + out = [] + for i in range(n): + im = Image.new("RGBA", (64, 64), (0, 0, 0, 0)) + x = 10 + (i * shift) % 30 + for y in range(20, 50): + for xx in range(x, x + 12): + im.putpixel((xx, y), (200, 60, 60, 255)) + out.append(im) + return out + + +def test_motion_scale_is_zero_for_a_frozen_sequence(): + """整段冻结时死帧判据一帧都报不出——两条判据都是相对的，d 全为 0 时 + `0 < 0` 一条都不成立。绝对尺度必须单独给一个，否则"每帧都一样"这种 + 最典型的坏产出在出参上完全看不见。""" + same = _frames(12, shift=0) + assert motion_scale(same) == 0.0 + assert len(dead_frame_indices(same)) == 0, "相对判据看不见整体没动 —— 正是要 motion_scale 的原因" + + +def test_motion_scale_positive_for_real_movement(): + assert motion_scale(_frames(12)) > 0.0 + + +def test_dead_frame_indices_returns_positions_not_a_mask(): + """跨出 ai_engine 的契约要"哪几帧"，不该让调用方拿 numpy 掩码去 argwhere。""" + idx = dead_frame_indices(_frames(10)) + assert isinstance(idx, tuple) + assert all(isinstance(i, int) for i in idx) + + +def test_loop_seam_returns_none_when_there_is_no_step_to_compare(): + """分母为 0 时返回 None 而不是 0.0——0.0 会被读成"完美闭环"， + 而真相是"没有可比的步长，这个数不可读"。""" + assert loop_seam(_frames(8, shift=0)) is None + assert loop_seam(_frames(1)) is None + + +def test_loop_seam_measures_the_gap_between_last_and_first(): + seam = loop_seam(_frames(10)) + assert seam is not None and seam >= 0.0 + + +def test_quality_fields_are_independent(): + """三个字段互不可推导：全同帧的 motion_scale=0 而 dead_frames 为空， + 两者若能互推，这一组断言不可能同时成立。""" + q = ActionQuality(motion_scale=0.0, dead_frames=(), loop_seam=None) + assert q.motion_scale == 0.0 and q.dead_frames == () and q.loop_seam is None diff --git a/backend/tests/test_matte_provider.py b/backend/tests/test_matte_provider.py new file mode 100644 index 00000000..0ac02076 --- /dev/null +++ b/backend/tests/test_matte_provider.py @@ -0,0 +1,442 @@ +"""OnnxU2NetMatteProvider 契约测试(不加载模型 / 不联网:构造 + 协议合规)。""" + +from windup_framework.providers import MatteProvider, OnnxU2NetMatteProvider + + +def test_onnx_matte_satisfies_matte_provider_protocol(): + # 运行时可检查协议:有 cutout 即满足 MatteProvider(server/ai_engine 依赖此契约) + provider = OnnxU2NetMatteProvider(model_path="/nonexistent/u2netp.onnx") + assert isinstance(provider, MatteProvider) + assert callable(provider.cutout) + + +def test_onnx_matte_lazy_no_model_load_on_construct(): + # 构造不触发下载 / 会话创建(惰性),模型缺失也不报错 + provider = OnnxU2NetMatteProvider(model_path="/nonexistent/u2netp.onnx") + assert provider._session is None + + +# ── 底色清理（2026-08-07 实测挣得）──────────────────────────────────────────── + + +def _rgb(w, h, bg, blob=None): + import numpy as np + a = np.zeros((h, w, 3), dtype=np.float32) + a[:, :] = bg + if blob: + (x0, y0, x1, y1), c = blob + a[y0:y1, x0:x1] = c + return a + + +def test_flat_background_is_killed_but_subject_untouched(): + """纯色底 → 系数 0（会被清掉）；主体色 → 系数 1（一像素不动）。""" + from windup_framework.providers.matte import _flat_bg_penalty + + bg = (222, 41, 124) # 实测的玫红底 + fur = (222, 130, 70) # 铁锈橙毛：与底色红通道相同，欧氏距离仅约 104 + a = _rgb(80, 60, bg, blob=((20, 15, 60, 45), fur)) + p = _flat_bg_penalty(a) + assert p[2, 2] == 0.0, "四角纯背景必须被判为 0" + assert p[30, 40] == 1.0, "橙毛必须完全不受影响 —— 宽阈值会把它反解成绿色" + + +def test_enclosed_background_gap_is_killed(): + """被主体围住的背景空隙也要清掉 —— u2netp 对闭合区域天然失灵。""" + from windup_framework.providers.matte import _flat_bg_penalty + + bg = (222, 41, 124) + a = _rgb(80, 60, bg, blob=((16, 16, 64, 44), (100, 120, 140))) # 避开取样用的 12×12 角落 + a[24:34, 30:50] = bg # 主体内部挖一个洞，填回底色 + p = _flat_bg_penalty(a) + assert p[30, 40] == 0.0, "闭合空隙里的底色必须被清掉" + assert p[20, 20] == 1.0, "洞外的主体不受影响" + + +def test_non_flat_background_disables_cleanup_entirely(): + """底色不均匀时一律不清理 —— 宁可漏，不可误伤。""" + import numpy as np + + from windup_framework.providers.matte import _flat_bg_penalty + + rng = np.random.default_rng(0) + noisy = rng.uniform(0, 255, (60, 80, 3)).astype(np.float32) + assert (_flat_bg_penalty(noisy) == 1.0).all() + + +def test_cleanup_only_subtracts_never_adds_subject(): + """系数恒在 [0,1] —— 只做减法，最坏情况是少清理，不会凭空造出主体。""" + from windup_framework.providers.matte import _flat_bg_penalty + + a = _rgb(40, 40, (0, 255, 0), blob=((5, 5, 35, 35), (200, 60, 60))) + p = _flat_bg_penalty(a) + assert p.min() >= 0.0 and p.max() <= 1.0 + + +def test_missing_onnxruntime_raises_instead_of_guessing_background(): + """装不上就报出来，不能回落到"猜四角主色"——白底浅色角色会被抠穿。""" + import builtins + + import pytest + + from windup_framework.providers.matte import OnnxU2NetMatteProvider + + real = builtins.__import__ + + def blocked(name, *a, **k): + if name == "onnxruntime": + raise ImportError("blocked for test") + return real(name, *a, **k) + + builtins.__import__ = blocked + try: + with pytest.raises(RuntimeError, match="onnxruntime"): + OnnxU2NetMatteProvider()._get_session() + finally: + builtins.__import__ = real + + +# ── 视频帧的最外圈是编码器伪影，不是底色（2026-08-10 实测挣得）────────────── + + +def test_edge_artifact_row_does_not_disable_cleanup(): + """最外一行/列常是编码器伪影：贴边采样会把它算进"底色是否均匀"， + 于是整帧被判"底不均匀"而跳过清理——修复在真实路径上等于从不生效。 + + 实测 9 段真 i2v × 16 帧 = 144 帧，贴边采样时 26 帧（18%）因此误跳； + 往里让 2px 后归零。 + """ + import numpy as np + + from windup_framework.providers.matte import _flat_bg_penalty + + bg = (222, 41, 124) + a = np.zeros((80, 80, 3), dtype=np.float32) + a[:, :] = bg + a[:, -1] = (0, 0, 0) # 最右一列纯黑：典型的编码器边缘伪影 + a[0, :] = (180, 30, 100) # 最顶一行偏暗 + p = _flat_bg_penalty(a) + assert p[40, 40] == 0.0, "跳过最外圈后应认出这是纯色底并清理；贴边采样会误判为不均匀" + + +def test_tiny_image_degrades_to_no_cleanup_rather_than_guessing(): + """图小到四角采样块会盖住主体时，采出来的"底色"其实混了主体色， + 此时守卫判"底不均匀"、整体跳过清理。 + + 这是**安全的退化方向**：清理只做减法，跳过等于少清一点；反过来若强行按 + 混了主体色的 key 去清，会把主体本身当背景抠掉——本项目宁可漏，不可误伤。 + """ + import numpy as np + + from windup_framework.providers.matte import _flat_bg_penalty + + a = np.zeros((20, 20, 3), dtype=np.float32) + a[:, :] = (0, 255, 0) + a[8:12, 8:12] = (200, 60, 60) # 主体落在四角采样块的重叠区 + assert (_flat_bg_penalty(a) == 1.0).all(), "采样不可靠时必须整体跳过，而不是按脏 key 清理" + + +# ── 封闭空洞填充（2026-08-11 在 121 帧真实走路视频帧上实测挣得）────────────────── +# +# 背景:交付帧放大看,主体内部会有透明洞(背景直接透出来)。实测拆开成因: +# · u2netp 自身在主体内部造的洞:8 帧抽样里 6 帧为 0 —— 不是主要成因; +# · 键控误杀:_flat_bg_penalty 每帧杀掉 820~2346 个 u2netp 判为主体的像素 —— +# 浅肤色 (243,221,200) 到灰底 (219,219,220) 的欧氏距离只有 31.3,窄于 _KEY_KILL=38。 +# 这些被误杀的像素被主体围住,正是"封闭空洞",填回来即修复。 +# +# 但**只按"不与画面边界连通"判定会把两腿之间填实**:迈步相里两只靴子在下方交叠, +# 把腿间空隙彻底封死。实测 121 帧中 80 帧存在这种封闭的底色空隙,共 25173 像素; +# 朴素版(只判连通性)把这 25173 像素全部填成主体(最惨单帧 3172 像素,两腿焊死), +# 加了颜色守卫后填掉 0 像素。下面的用例把这条守住。 + + +def _walk_frame(*, gap_closed: bool, hole: bool = False, eroded_leg: bool = False): + """造一帧"迈步相":灰底 + 躯干 + 两条腿 + 腿间底色空隙。 + + ``gap_closed=True`` 时靴子在下方交叠、把腿间空隙封死(实测 80/121 帧是这形状)。 + 颜色取实测值:底 (219,219,220)、浅肤 (243,221,200)(两者距离 31.3,窄于 _KEY_KILL)。 + 返回 (rgb float32, alpha float32)。 + """ + import numpy as np + + bg, skin, cloth = (219, 219, 220), (243, 221, 200), (110, 130, 100) + h, w = 64, 64 + rgb = np.full((h, w, 3), bg, dtype=np.float32) + alpha = np.zeros((h, w), dtype=np.float32) + + def paint(y0, y1, x0, x1, color): + rgb[y0:y1, x0:x1] = color + alpha[y0:y1, x0:x1] = 1.0 + + paint(8, 32, 20, 44, cloth) # 躯干 + paint(32, 52, 20, 28, skin) # 后腿 + paint(32, 52, 36, 44, skin) # 前腿 + if gap_closed: + paint(52, 58, 20, 44, cloth) # 靴子交叠 → 腿间空隙被封死 + else: + paint(52, 58, 20, 28, cloth) # 两只靴子分开 → 空隙通到画面底边 + paint(52, 58, 36, 44, cloth) + if hole: + alpha[14:20, 28:36] = 0.0 # 躯干内部的洞:颜色还是衣服色 + if eroded_leg: + alpha[36:46, 22:26] = 0.0 # 腿内部被键控误杀的一条:颜色是浅肤色 + return rgb, alpha + + +def _gap_slice(): + """腿间空隙区域(rgb 一直是底色,alpha 一直应为 0)。""" + return (slice(32, 52), slice(28, 36)) + + +def test_enclosed_hole_in_subject_is_filled(): + """被主体围住、颜色不是底色的透明块 = 洞,填成主体。""" + from windup_framework.providers.matte import _fill_enclosed_holes + + rgb, alpha = _walk_frame(gap_closed=True, hole=True) + out = _fill_enclosed_holes(alpha, rgb) + assert (out[14:20, 28:36] == 1.0).all(), "躯干内部的洞必须被填成主体" + + +def test_keyed_out_skin_inside_leg_is_filled(): + """被 _flat_bg_penalty 误杀的浅肤色(实测每帧 820~2346 px)要能填回来。""" + from windup_framework.providers.matte import _fill_enclosed_holes + + rgb, alpha = _walk_frame(gap_closed=True, eroded_leg=True) + out = _fill_enclosed_holes(alpha, rgb) + assert (out[36:46, 22:26] == 1.0).all(), "浅肤色距底色 31.3,不是底色,必须填回主体" + + +def test_closed_leg_gap_is_never_filled(): + """**核心回归**:靴子交叠把腿间空隙封死时,它照样不能被填 —— 否则两腿焊在一起。 + + 实测:只判"不与边界连通"的朴素版在这里会把整块空隙填掉(121 帧共 25173 px)。 + """ + from windup_framework.providers.matte import _fill_enclosed_holes + + rgb, alpha = _walk_frame(gap_closed=True) + ys, xs = _gap_slice() + assert (alpha[ys, xs] == 0.0).all(), "前提:空隙本来是透明的" + out = _fill_enclosed_holes(alpha, rgb) + assert (out[ys, xs] == 0.0).all(), "腿间空隙整块是底色,一个像素都不能填" + + +def test_open_leg_gap_is_never_filled(): + """空隙通到画面底边时同样不能填 —— 这条也钉死"绝不能按行/按列填"。 + + 按行填会看到"这一行左右都是主体"就把中间填上,正是这里要拦的。 + """ + from windup_framework.providers.matte import _fill_enclosed_holes + + rgb, alpha = _walk_frame(gap_closed=False) + ys, xs = _gap_slice() + out = _fill_enclosed_holes(alpha, rgb) + assert (out[ys, xs] == 0.0).all(), "与边界连通的空隙不是洞" + assert (out == alpha).all(), "没有洞的帧必须逐像素不变" + + +def test_border_touching_transparent_area_is_never_filled(): + """贴着画幅边缘的透明区域不是洞 —— 哪怕它的颜色一点也不像底色。 + + 真实场景:i2v 出的帧经常把角色下半身裁出画,两腿之间是一条暗投影(不是干净底色), + 这条投影只从画幅下沿通向画外,左右被两条腿封死。只靠"颜色像不像底色"判断会把 + 它当成洞、填成主体(两腿又焊上了),所以"从边界出发"这条种子必须保留。 + """ + from windup_framework.providers.matte import _fill_enclosed_holes + + rgb, alpha = _walk_frame(gap_closed=False) + for x0, x1 in ((20, 28), (36, 44)): # 两条腿一直延到画幅下沿 + rgb[52:64, x0:x1] = (110, 130, 100) + alpha[52:64, x0:x1] = 1.0 + rgb[32:64, 28:36] = (60, 55, 50) # 腿间暗投影:远离底色 + alpha[32:64, 28:36] = 0.0 # 只从下沿通向画外,左右被腿封死 + + out = _fill_enclosed_holes(alpha, rgb) + assert (out[32:64, 28:36] == 0.0).all(), "连到画幅边界的透明区域一律不是洞" + + +def test_frame_without_holes_is_pixel_identical(): + """没有洞 → 逐像素不变(防回归硬指标)。""" + import numpy as np + + from windup_framework.providers.matte import _fill_enclosed_holes + + rgb, alpha = _walk_frame(gap_closed=True) + out = _fill_enclosed_holes(alpha, rgb) + assert np.array_equal(out, alpha) + + +def test_fill_only_adds_alpha_never_removes(): + """只做加法:alpha 绝不被改小,改动值只能是 1.0 —— 填洞不该顺手抠掉别的。""" + from windup_framework.providers.matte import _fill_enclosed_holes + + rgb, alpha = _walk_frame(gap_closed=True, hole=True, eroded_leg=True) + out = _fill_enclosed_holes(alpha, rgb) + assert (out >= alpha).all() + assert (out[out != alpha] == 1.0).all() + + +def test_non_flat_background_disables_fill_entirely(): + """底色不均匀 → 无从判断哪块是真空隙,一律不填(与键控清理同一条纪律)。""" + import numpy as np + + from windup_framework.providers.matte import _fill_enclosed_holes + + rgb, alpha = _walk_frame(gap_closed=True, hole=True) + rng = np.random.default_rng(0) + noisy = rng.uniform(0, 255, rgb.shape).astype(np.float32) + out = _fill_enclosed_holes(alpha, noisy) + assert np.array_equal(out, alpha), "底不是纯色时必须整帧不动" + + +def test_bg_key_returns_none_when_background_is_not_flat(): + """底色真相源:不均匀时返回 None,键控与填洞都据此停手。""" + import numpy as np + + from windup_framework.providers.matte import _bg_key + + rng = np.random.default_rng(1) + assert _bg_key(rng.uniform(0, 255, (40, 40, 3)).astype(np.float32)) is None + assert _bg_key(np.full((40, 40, 3), 219, dtype=np.float32)) is not None + + +def test_spread_is_four_connected_not_scanline(): + """扩散必须是真 4-邻接连通:L 形走廊要能拐弯走通,断开的孤岛不能被沾到。 + + 只做行传播(或只做列传播)都会让 L 形的另一条臂走不通,这条用例把两个方向都钉死。 + """ + import numpy as np + + from windup_framework.providers.matte import _spread + + region = np.zeros((20, 20), dtype=bool) + region[2, 2:18] = True # 横臂 + region[2:18, 17] = True # 竖臂(拐弯) + island = (15, 3) + region[island] = True # 孤岛:与走廊不连通 + seed = np.zeros_like(region) + seed[2, 2] = True + + reach = _spread(seed, region) + assert reach[2, 17], "横臂尽头要走通(需要行传播)" + assert reach[17, 17], "竖臂尽头要走通(需要列传播)" + assert not reach[island], "不连通的孤岛绝不能被标记为可达" + + +def test_spread_matches_bruteforce_bfs_on_random_masks(): + """与逐像素 BFS 逐点等价 —— 向量化只是为了快,不能改语义。""" + from collections import deque + + import numpy as np + + from windup_framework.providers.matte import _spread + + def bfs(seed, region): + h, w = region.shape + out = np.zeros_like(region) + q = deque() + for y, x in zip(*np.nonzero(seed & region), strict=True): + out[y, x] = True + q.append((y, x)) + while q: + cy, cx = q.popleft() + for dy, dx in ((1, 0), (-1, 0), (0, 1), (0, -1)): + ny, nx = cy + dy, cx + dx + if 0 <= ny < h and 0 <= nx < w and region[ny, nx] and not out[ny, nx]: + out[ny, nx] = True + q.append((ny, nx)) + return out + + rng = np.random.default_rng(7) + for _ in range(25): + h, w = int(rng.integers(3, 30)), int(rng.integers(3, 30)) + region = rng.random((h, w)) < rng.uniform(0.3, 0.9) + seed = rng.random((h, w)) < 0.05 + assert (_spread(seed, region) == bfs(seed, region)).all() + + +# ── cutout 的装配顺序（不碰真模型）───────────────────────────────────────── +# +# 真实推理需要 4.7MB 的 onnx 权重，CI 里既下不到也不该下。但 cutout 本身的**装配顺序** +# 是有语义的，可以用一个假 session 覆盖： +# 预测 mask → 乘键控清理系数 → 填封闭空洞 → 合成 RGBA +# 顺序错了会静默出错结果：先填洞再清理，会把刚填上的像素又清掉。 + + +class _FakeSession: + """假 onnxruntime session：返回一个中间为主体的 mask。""" + + class _In: + name = "input" + + def get_inputs(self): + return [self._In()] + + def run(self, _out, feed): + import numpy as np + + t = next(iter(feed.values())) + h, w = t.shape[2], t.shape[3] + m = np.zeros((1, 1, h, w), dtype="float32") + m[:, :, h // 4 : h * 3 // 4, w // 4 : w * 3 // 4] = 1.0 + return [m] + + +def _provider_with_fake_session(monkeypatch): + from windup_framework.providers.matte import OnnxU2NetMatteProvider + + p = OnnxU2NetMatteProvider() + monkeypatch.setattr(p, "_get_session", lambda: _FakeSession()) + return p + + +def _png(w=64, h=64, color=(220, 220, 220)): + import io + + from PIL import Image + + buf = io.BytesIO() + Image.new("RGB", (w, h), color).save(buf, "PNG") + return buf.getvalue() + + +def test_cutout_outputs_rgba_png_with_alpha(monkeypatch): + import io + + from PIL import Image + + out = _provider_with_fake_session(monkeypatch).cutout(_png()) + im = Image.open(io.BytesIO(out)) + assert im.format == "PNG" and im.mode == "RGBA" + assert im.size == (64, 64) + + +def test_cutout_keeps_rgb_untouched(monkeypatch): + """抠图只动 alpha。改 RGB 会让后续像素化锁色板取到被改过的颜色。""" + import io + + import numpy as np + from PIL import Image + + src = _png(color=(31, 41, 59)) + out = _provider_with_fake_session(monkeypatch).cutout(src) + a = np.asarray(Image.open(io.BytesIO(out))) + b = np.asarray(Image.open(io.BytesIO(src)).convert("RGB")) + assert np.array_equal(a[:, :, :3], b), "RGB 通道被改了" + + +def test_cutout_applies_flat_bg_cleanup_before_filling_holes(monkeypatch): + """顺序：清理 → 填洞。反过来会把刚填上的像素又清掉，且不报错。 + + 用调用顺序断言而不是像素结果 —— 结果层面两种顺序在简单图上可能相同， + 那样的用例杀不掉顺序颠倒这个变异。 + """ + import windup_framework.providers.matte as M + + order: list[str] = [] + real_pen, real_fill = M._flat_bg_penalty, M._fill_enclosed_holes + monkeypatch.setattr(M, "_flat_bg_penalty", + lambda rgb: (order.append("clean"), real_pen(rgb))[1]) + monkeypatch.setattr(M, "_fill_enclosed_holes", + lambda a, rgb: (order.append("fill"), real_fill(a, rgb))[1]) + _provider_with_fake_session(monkeypatch).cutout(_png()) + assert order == ["clean", "fill"], order diff --git a/backend/tests/test_oneshot.py b/backend/tests/test_oneshot.py new file mode 100644 index 00000000..2307a030 --- /dev/null +++ b/backend/tests/test_oneshot.py @@ -0,0 +1,177 @@ +"""一次性动作抽帧(裁动作区间 / 跳跃状态切段)测试 —— 纯 CV,无需联网。""" + +import numpy as np +import pytest +from PIL import Image + +from windup_ai_engine.slicing import ( + find_motion_span, + first_action_end, + foot_line_series, + pick_oneshot, + split_jump_phases, +) + + +def _figure_at(y_bottom: int, size: int = 64, h: int = 20) -> Image.Image: + """在指定底边高度画一个方块"角色"(RGBA,其余透明)。""" + img = Image.new("RGBA", (size, size), (0, 0, 0, 0)) + arr = np.asarray(img).copy() + top = max(0, y_bottom - h) + arr[top:y_bottom, size // 2 - 4 : size // 2 + 4] = (200, 60, 60, 255) + return Image.fromarray(arr, "RGBA") + + +def _jump_sequence() -> list[Image.Image]: + """合成跳跃:静止 → 蹲(底边下移)→ 升 → 顶点 → 落 → 静止。""" + ground, low, apex = 50, 52, 30 + ys = [ground] * 3 + [low, low] + [44, 38, apex, apex, 38, 44] + [ground] * 3 + return [_figure_at(y) for y in ys] + + +def test_find_motion_span_trims_static_head_and_tail(): + frames = _jump_sequence() + start, end = find_motion_span(frames) + assert start >= 1 # 前面的静止帧被裁掉 + assert end <= len(frames) - 2 # 后面的静止帧被裁掉 + assert end > start + + +def test_pick_oneshot_returns_n_and_does_not_wrap(): + frames = _jump_sequence() + out = pick_oneshot(frames, 6) + assert len(out) == 6 + # 一次性动作不闭环:首尾姿态应不同(闭环的话会几乎一样) + first = np.asarray(out[0].convert("L"), float) + last = np.asarray(out[-1].convert("L"), float) + assert np.abs(first - last).mean() >= 0 + + +def test_foot_line_tracks_height(): + frames = _jump_sequence() + y = foot_line_series(frames) + assert y.argmin() in range(6, 10) # 最高点(y 最小)落在顶点附近 + assert y[0] > y.min() # 起始在地面,低于顶点 + + +def test_split_jump_phases_covers_all_frames_in_order(): + frames = _jump_sequence() + phases = split_jump_phases(frames) + assert "apex" in phases + idx = [i for seg in phases.values() for i in seg] + assert sorted(idx) == list(range(len(frames))) # 不重不漏 + # apex 段应在 rise 之后、fall 之前 + if "rise" in phases and "fall" in phases: + assert max(phases["rise"]) < min(phases["apex"]) + assert max(phases["apex"]) < min(phases["fall"]) + + +def test_split_jump_phases_short_input_is_safe(): + assert split_jump_phases([_figure_at(50)] * 3) + + +# ── 入参边界 ──────────────────────────────────────────────────────────────── +# 契约:返回长度恒等于 n;凡是给不出 n 帧的入参一律报错,绝不静默少给。 + + +def _bar_at(x: int, w: int = 8, size: int = 64) -> Image.Image: + """横向位移的方块,用来造挥击序列。位移刻意都 < 条宽 → 像素差与位移成正比、不饱和。""" + img = Image.new("RGBA", (size, size), (0, 0, 0, 0)) + arr = np.asarray(img).copy() + arr[20:50, x : x + w] = (200, 60, 60, 255) + return Image.fromarray(arr, "RGBA") + + +def _swing_sequence() -> list[Image.Image]: + """合成挥击:静止 → 加速横扫(最快的一跳是 16→22)→ 收势 → 静止。""" + xs = [10] * 4 + [11, 13, 16, 22, 25, 26] + [26] * 4 + return [_bar_at(x) for x in xs] + + +def _tail_action_sequence() -> list[Image.Image]: + """动作贴在尾部(视频在半空结束,没有静止收尾)—— 区间放宽时右边无处可长, + 必须把缺口退回左边,否则窗口不足 n 帧、只能靠重复帧凑数。""" + ys = [50] * 8 + [52, 52, 44, 38, 30, 30, 38, 44] + return [_figure_at(y) for y in ys] + + +def test_pick_oneshot_n1_takes_apex_for_airborne(): + """n=1 取关键姿势:腾空类应给顶点帧,而不是首帧(蓄力,和待机一个样)。""" + frames = _jump_sequence() + apex = int(np.argmin(foot_line_series(frames))) + out = pick_oneshot(frames, 1, kind="airborne") + assert len(out) == 1 + assert out[0] is frames[apex] + + +def test_pick_oneshot_n1_takes_impact_for_swing(): + """n=1 取关键姿势:挥击类应给"刚走完最快一跳"的命中帧,不是首帧/末帧。""" + frames = _swing_sequence() + out = pick_oneshot(frames, 1) + assert len(out) == 1 + assert out[0] is frames[7] # 16→22 是最快的一跳,落点即命中姿势 + assert out[0] is not frames[0] and out[0] is not frames[-1] + + +def test_pick_oneshot_rejects_non_positive_n(): + """n<=0 原本静默返回 [](零帧也算"成功"),现在必须报错。""" + frames = _jump_sequence() + for n in (0, -1): + with pytest.raises(ValueError, match="n 必须"): + pick_oneshot(frames, n) + + +def test_pick_oneshot_rejects_insufficient_source(): + """源帧不够 n 帧:报错并把两个数字都说清楚,不再原样返回一个短序列。""" + frames = _jump_sequence() + with pytest.raises(ValueError, match=r"源帧不足.*19.*14"): + pick_oneshot(frames, len(frames) + 5) + with pytest.raises(ValueError, match="源帧不足"): + pick_oneshot([], 4) + with pytest.raises(ValueError, match="源帧不足"): + pick_oneshot([_figure_at(50)], 2) + + +def test_pick_oneshot_returns_exactly_n_for_every_legal_n(): + """全量扫 n=1..len(frames):长度必须恒等于 n。 + + 修前 pick_oneshot(jump14, 12) 只回 9 帧 —— 动作区间被裁到 9 帧后直接原样返回, + 而下游 frame_durations 按实际长度现算时长,帧数与时长自洽,谁都看不出少了 3 帧。 + """ + for frames in (_jump_sequence(), _swing_sequence(), _tail_action_sequence()): + for kind in ("swing", "airborne"): + for n in range(1, len(frames) + 1): + assert len(pick_oneshot(frames, n, kind=kind)) == n, (kind, n) + for n in range(1, len(frames) + 1): # first_only=False 走另一条区间分支 + assert len(pick_oneshot(frames, n, first_only=False)) == n + + +def test_pick_oneshot_passthrough_when_n_equals_len(): + frames = _jump_sequence() + assert pick_oneshot(frames, len(frames)) is frames + + +def test_pick_oneshot_frames_are_distinct_source_frames_in_order(): + """源帧够 n 张时,返回的 n 帧必须**互不相同**且时间顺序不倒。 + + 长度对但夹着重复帧,是另一种"看起来成功"的错结果:时长表照样自洽,播出来是卡顿。 + 动作贴尾部的序列是这条的关键用例 —— 区间往右长不动,缺口只能退回左边补。 + """ + for frames in (_jump_sequence(), _swing_sequence(), _tail_action_sequence()): + for n in range(1, len(frames) + 1): + out = pick_oneshot(frames, n, kind="airborne") + pos = [next(i for i, f in enumerate(frames) if f is o) for o in out] + assert pos == sorted(pos), (n, pos) + assert len(set(pos)) == n, (n, pos) # 无重复帧 + + +def test_unknown_kind_is_rejected(): + """kind 拼错不能静默按 swing 处理 —— 判据用错会裁出"看起来对"的错区间。""" + frames = _jump_sequence() + with pytest.raises(ValueError, match="kind"): + pick_oneshot(frames, 6, kind="airbourne") + with pytest.raises(ValueError, match="kind"): + # first_only=False 不走 first_action_end,得靠 pick_oneshot 自己那道校验 + pick_oneshot(frames, 6, kind="airbourne", first_only=False) + with pytest.raises(ValueError, match="kind"): + first_action_end(frames, 0, 1, kind="airbourne") # 短区间(早返回)也要拦住 diff --git a/backend/tests/test_orchestrator_hardening.py b/backend/tests/test_orchestrator_hardening.py new file mode 100644 index 00000000..4b4b9ed2 --- /dev/null +++ b/backend/tests/test_orchestrator_hardening.py @@ -0,0 +1,318 @@ +"""编排层的加固用例。 + +EventBus 的键是 ``(project_id, task_id)``（主线 #110：同一 task_id 在不同项目下互不 +串流）。本文件里的 project_id 取一个固定值即可 —— 这些用例验的是队列/loop 行为， +项目隔离本身由 test_generation_api.py 的专用用例覆盖。 + +原始标题：编排层的五处加固（2026-08-10 机器审逮到，逐条锁死）。 + +共同点：全部在**测试全绿的情况下**存在——注入桩的测试走不到真实装配路径， +mock 的 EventBus 不涉及跨线程，请求模型的上界靠"没人会填大数"活着。 +""" +from __future__ import annotations + +import asyncio +import threading + +import pytest + +from windup_app.server.orchestrator._fetch import ( + MAX_FETCH_BYTES, + FetchNotAllowed, + fetch_own_media, +) +from windup_app.server.orchestrator.model import TaskStatus +from windup_app.web.api.generation import ( + _TERMINAL_EVENTS, + CharacterActionGenerateRequest, + CharacterImageGenerateRequest, + _EventBus, +) + + +# ── ① 真实装配路径不能引用已删除的路线 ──────────────────────────────────── + + +def test_real_generator_assembly_covers_every_declared_route(): + """曾多装一个 PROC_IDLE：该枚举与 ProcIdleStrategy 都已随「程序化待机放弃」 + 删除，而装配那行留着，于是每个动作任务在 import 期 AttributeError。 + + 注入 generator 的测试走不到这条路径 —— 所以这条必须直接调真实装配。 + """ + from windup_common.models import GenRoute + from windup_app.server.orchestrator.executor import ActionTaskExecutor + + gen = ActionTaskExecutor()._get_generator() + wired = set(gen._by_route) + assert wired == set(GenRoute), ( + f"GenRoute 声明了 {sorted(r.value for r in GenRoute)}，" + f"装配了 {sorted(r.value for r in wired)} —— 漏装的路线一被请求就崩" + ) + + +# ── ② 服务端取图必须白名单 ──────────────────────────────────────────────── + + +@pytest.mark.parametrize("evil", [ + "http://127.0.0.1:8000/auth/me", # 打回自己，绕过鉴权中间件 + "http://169.254.169.254/latest/meta-data/", # 云实例元数据服务 + "http://10.0.0.5/internal", # 私网探测 + "file:///etc/passwd", + "http://[::1]:8000/", +]) +def test_server_side_fetch_rejects_non_own_urls(evil: str, monkeypatch): + """URL 来自已认证请求的请求体，直接 httpx.get 等于把服务器当跳板。""" + import windup_app.server.orchestrator._fetch as F + + monkeypatch.setattr(F.storage_settings, "bucket_domain", "https://cdn.example.com") + with pytest.raises(FetchNotAllowed): + fetch_own_media(evil) + + +def test_server_side_fetch_refuses_when_storage_domain_unset(monkeypatch): + """下载域名没配时不能"放行一切"——那等于白名单形同虚设。""" + import windup_app.server.orchestrator._fetch as F + + monkeypatch.setattr(F.storage_settings, "bucket_domain", "") + with pytest.raises(FetchNotAllowed, match="未配置"): + fetch_own_media("https://cdn.example.com/a.png") + + +def test_prefix_match_is_not_fooled_by_a_lookalike_host(monkeypatch): + """`cdn.example.com.evil.com` 不能因为前缀相似而通过。""" + import windup_app.server.orchestrator._fetch as F + + monkeypatch.setattr(F.storage_settings, "bucket_domain", "https://cdn.example.com") + with pytest.raises(FetchNotAllowed): + fetch_own_media("https://cdn.example.com.evil.com/a.png") + + +def test_fetch_size_cap_is_bounded(): + """上限存在且是个有限的正数——无上限时一个指向大文件的 URL 就能吃光 worker 内存。""" + assert 0 < MAX_FETCH_BYTES <= 64 * 1024 * 1024 + + +# ── ③ 终态事件名必须与 SSE 契约一致 ────────────────────────────────────── + + +@pytest.mark.parametrize(("status", "expected"), [ + (TaskStatus.COMPLETED, "completed"), + (TaskStatus.FAILED, "failed"), + (TaskStatus.RUNNING, "task_update"), +]) +def test_terminal_states_publish_terminal_event_names(status, expected, monkeypatch): + """一律发 task_update 的话，stream 的终态判断永不成立：客户端收到 completed + 后连接仍开着，而端点带 retry: 3000，浏览器每 3 秒重连、重收同一条 completed。 + + 走**真实的 _publish_task_update 调用路径**，不读 _STATUS_EVENT 字典 —— 只断言 + 字典内容的话，把 `event = _STATUS_EVENT.get(...)` 改成 `event = "task_update"` + 测试照样绿（2026-08-10 变异测试逮到这条是摆设）。 + """ + import windup_app.server.orchestrator.task_repo as R + from windup_app.server.orchestrator.model import GenerationTask, GenerationType + + sent: list[str] = [] + + class _Bus: + def publish(self, project_id, task_id, event, data): + sent.append(event) + + monkeypatch.setattr(R, "_event_bus", _Bus()) + # 必须带 project_id：EventBus 按 (project_id, task_id) 索引，_publish_task_update + # 对 project_id 为空的任务会记 warning 并早退（发到没人听的键上等于静默失败）。 + R._publish_task_update(1, GenerationTask( + id=1, user_id=1, project_id=42, + task_type=GenerationType.CHARACTER_ACTION, status=status, + )) + assert sent == [expected] + + +def test_every_terminal_event_name_is_recognised_by_the_stream(): + """两边是一套契约的两半，任何一边改了名字必须让另一边失败。""" + from windup_app.server.orchestrator.task_repo import _STATUS_EVENT + + assert set(_STATUS_EVENT.values()) == _TERMINAL_EVENTS + + +# ── ④ EventBus 跨线程投递 ──────────────────────────────────────────────── + + +def test_publish_from_another_thread_delivers(): + """executor 在 daemon thread 里跑，队列属于处理 SSE 请求的那个 loop。 + + 诚实说明本用例的强度：它只证明跨线程发布**能到达**订阅者，**证不出** + call_soon_threadsafe 是必需的 —— 实测在这个单队列场景里，裸 put_nowait + 跨线程也能被 get() 取到（CPython 的 Queue.get 在有元素时走快路径、不等唤醒）。 + + call_soon_threadsafe 仍然要留：asyncio.Queue 的文档明说它不是线程安全的， + 上面那个"能取到"是实现细节而非保证——多个 waiter、队列非空判定与唤醒之间 + 的竞态都可能让它失效。真正的保证由下一条用例（订阅记录 loop）间接锁住。 + + 先等发布线程真的调完再取（用一个 future 同步），否则测的是"抢跑运气"而不是投递： + publish 走的是跨 loop 分支，marshal 回来要等 loop 一次迭代。 + """ + bus = _EventBus() + + async def scenario(): + queue = await bus.subscribe(42, 7) + loop = asyncio.get_running_loop() + published = loop.create_future() + + def publish_from_thread(): + bus.publish(42, 7, "completed", {"id": 7}) + loop.call_soon_threadsafe(published.set_result, None) + + threading.Thread(target=publish_from_thread, daemon=True).start() + await asyncio.wait_for(published, timeout=3.0) + return await asyncio.wait_for(queue.get(), timeout=3.0) + + event, data = asyncio.run(scenario()) + assert event == "completed" and data["id"] == 7 + + +def test_subscription_records_its_owning_loop(): + """订阅必须记下所属 loop —— 这是跨线程安全投递的前提。 + + 只存 queue 的话，publish 无从知道该把入队动作 marshal 回哪个 loop； + 不同订阅者可能来自不同 loop（多 worker / 测试里的临时 loop），存一个全局 + loop 也不行。本用例锁住"每个订阅都带着自己的 loop"这个结构。 + """ + bus = _EventBus() + + async def scenario(): + q = await bus.subscribe(42, 11) + subs = bus._queues[(42, 11)] + assert len(subs) == 1 + queue, loop = subs[0] + assert queue is q + assert loop is asyncio.get_running_loop() + + asyncio.run(scenario()) + + +def test_publish_to_a_closed_loop_is_dropped_not_raised(): + """客户端断连后请求 loop 已关闭。此时发布应静默丢弃——任务状态本身已落库， + 重连后靠 GET /tasks/{id} 取；让它抛异常会把后台任务整个带崩。 + """ + bus = _EventBus() + + async def sub(): + return await bus.subscribe(42, 9) + + loop = asyncio.new_event_loop() + queue = loop.run_until_complete(sub()) + loop.close() + assert queue is not None + bus.publish(42, 9, "completed", {"id": 9}) # 不应抛 + + +def test_unsubscribe_removes_only_that_queue(): + """订阅记的是 (queue, loop) 元组，退订不能顺手把同一任务的其他订阅者删掉。""" + bus = _EventBus() + + async def scenario(): + q1 = await bus.subscribe(42, 3) + q2 = await bus.subscribe(42, 3) + await bus.unsubscribe(42, 3, q1) + bus.publish(42, 3, "task_update", {"n": 1}) + got = await asyncio.wait_for(q2.get(), timeout=2.0) + assert got[1]["n"] == 1 + assert q1.empty() + + asyncio.run(scenario()) + + +# ── ⑤ 付费循环必须有上界 ───────────────────────────────────────────────── + + +def test_num_images_is_bounded_at_the_contract_layer(): + """num_images 是 provider 调用次数的循环上界：一个已认证请求填个大数就能 + 绕过按请求计的限流，把成本拉到无上限。 + """ + with pytest.raises(ValueError): + CharacterImageGenerateRequest(project_id=42, prompt="x", num_images=10_000) + with pytest.raises(ValueError): + CharacterImageGenerateRequest(project_id=42, prompt="x", num_images=0) + assert CharacterImageGenerateRequest(project_id=42, prompt="x", num_images=2).num_images == 2 + + +def test_image_dimensions_are_bounded(): + with pytest.raises(ValueError): + CharacterImageGenerateRequest(project_id=42, prompt="x", width=100_000) + with pytest.raises(ValueError): + CharacterImageGenerateRequest(project_id=42, prompt="x", height=1) + + +def test_num_frames_is_bounded(): + """帧数决定抽帧与逐帧抠图的工作量。""" + with pytest.raises(ValueError): + CharacterActionGenerateRequest(project_id=42, character_id=1, action_type="walk", num_frames=100_000) + with pytest.raises(ValueError): + CharacterActionGenerateRequest(project_id=42, character_id=1, action_type="walk", num_frames=0) + ok = CharacterActionGenerateRequest(project_id=42, character_id=1, action_type="walk", num_frames=16) + assert ok.num_frames == 16 + + +# ── ⑥ 请求里的尺寸必须真的生效（2026-08-10 对抗复查）──────────────────────── + + +def _png(w: int, h: int) -> bytes: + """带细节的图。纯色图在 NEAREST 与 LANCZOS 下产出完全相同,拿它验重采样是无效仪器 + (2026-08-10 第一版就是这么写的,测试立刻变红)。这里用 8px 棋盘格。""" + import io + + import numpy as np + from PIL import Image + + y, x = np.mgrid[0:h, 0:w] + checker = (((x // 8) + (y // 8)) % 2 * 255).astype("uint8") + arr = np.dstack([checker, 255 - checker, checker, np.full((h, w), 255, "uint8")]) + buf = io.BytesIO() + Image.fromarray(arr, "RGBA").save(buf, "PNG") + return buf.getvalue() + + +@pytest.mark.parametrize(("want_w", "want_h"), [(512, 512), (256, 384), (1024, 1024)]) +def test_requested_image_size_is_actually_applied(want_w, want_h): + """入口收下 width/height 并校验过，但 ImageProvider.gen_image 没有尺寸参数。 + + 此前模型出多大就返多大：调用方要 512×512、拿到 1024×1024，而请求被接受了 —— + 又一个"接了不履约"的字段。本用例锁住"要多大就得多大"。 + """ + import io + + from PIL import Image + + from windup_app.server.orchestrator.executor import ImageTaskExecutor + from windup_app.server.orchestrator.model import CharacterImageInput + + class _Gen: + def gen_image(self, prompt, refs): + return _png(1024, 1024) # 模型固定出 1024² + + got: list[bytes] = [] + ex = ImageTaskExecutor(image=_Gen(), upload=lambda b: (got.append(b), "u")[1]) + ex._produce_image( + CharacterImageInput(prompt="knight", width=want_w, height=want_h, num_images=1), + _constraints(), + ) + assert Image.open(io.BytesIO(got[0])).size == (want_w, want_h) + + +def test_sprite_frames_and_master_use_different_resampling(): + """序列帧是像素画,必须 NEAREST;全彩母版用 NEAREST 缩图会明显锯齿。 + + 只断言两条路径产出不同 —— 同一张图两种重采样若字节相同,说明 smooth 参数没接上。 + """ + from windup_app.server.orchestrator.executor import _fit_to + + src = _png(1024, 1024) + assert _fit_to(src, 256, 256, smooth=False) != _fit_to(src, 256, 256, smooth=True) + + +def _constraints(): + """最小项目约束(本文件只关心尺寸这条链路)。""" + from windup_app.server.orchestrator.executor import _load_constraints # noqa: F401 + from windup_app.server.orchestrator.executor import ProjectConstraints + + return ProjectConstraints() diff --git a/backend/tests/test_pack_align.py b/backend/tests/test_pack_align.py new file mode 100644 index 00000000..e1d89653 --- /dev/null +++ b/backend/tests/test_pack_align.py @@ -0,0 +1,114 @@ +"""align_bottom_center 的交付画布几何(2026-08-11 挣得)。 + +为什么要这组用例:交付帧一直写死出 256×256 方形,而项目的 sprite 尺寸是 +``sprite_width×sprite_height``(32~2048,可非方)。上层拿到 256 的帧再 ``_fit_to`` +到项目尺寸,用的是 ``Image.thumbnail`` —— **它只缩不放**:项目要 512 时帧根本不会被 +放大,而是原尺寸居中贴进 512 画布,刚对齐好的脚线 0.92 被挪到 0.709(实测),角色不站 +在地上了。所以引擎必须能一次出到目标尺寸,而不是让上层再缩一次。 +""" + +import numpy as np +from PIL import Image + +from windup_ai_engine.postprocess.pack import align_bottom_center + +FILL_H = 0.62 # 与 pack.align_bottom_center 的默认值一致 +FOOT_LINE = 0.92 + + +def _frames(n=4, w=640, h=480, bh=300, bw=60): + """造一组"角色在画布里漂移"的帧(align 要消掉的正是这个漂移)。""" + out = [] + for i in range(n): + a = np.zeros((h, w, 4), dtype=np.uint8) + x0, y0 = 200 + i * 7, 60 + i * 5 + a[y0:y0 + bh, x0:x0 + bw] = (200, 80, 60, 255) + out.append(Image.fromarray(a, "RGBA")) + return out + + +def _subject(img: Image.Image): + """返回 (高, 脚线比例, 水平中心比例)。""" + a = np.asarray(img)[:, :, 3] + ys, xs = np.nonzero(a > 128) + w, h = img.size + return int(ys.max() - ys.min() + 1), (int(ys.max()) + 1) / h, (int(xs.min()) + int(xs.max())) / 2 / w + + +def test_default_canvas_is_256_square_with_foot_line_geometry(): + """默认仍是 256 方形,脚线 0.92、主体占高 0.62、水平居中。""" + out = align_bottom_center(_frames(), ref_height=300.0) + assert out[0].size == (256, 256) + height, foot, center = _subject(out[0]) + assert abs(height - 256 * FILL_H) <= 2 + assert abs(foot - FOOT_LINE) <= 0.01 + assert abs(center - 0.5) <= 0.01 + + +def test_omitting_cell_h_is_pixel_identical_to_square_cell(): + """不传 cell_h == 传 cell_h=cell —— 默认行为一个像素都不许变。""" + src = _frames() + a = align_bottom_center(src, ref_height=300.0) + b = align_bottom_center(src, ref_height=300.0, cell_h=256) + for x, y in zip(a, b, strict=True): + assert np.array_equal(np.asarray(x), np.asarray(y)) + + +def test_doubling_cell_doubles_subject_height(): + """指定 512 时交付帧主体高度翻倍 —— 这正是"交付帧太小"的修法。""" + src = _frames() + small = align_bottom_center(src, ref_height=300.0) + big = align_bottom_center(src, ref_height=300.0, cell=512) + assert big[0].size == (512, 512) + h_small = _subject(small[0])[0] + h_big = _subject(big[0])[0] + assert abs(h_big / h_small - 2.0) < 0.05, f"期望约翻倍,实际 {h_small} → {h_big}" + + +def test_non_square_canvas_applies_each_axis_to_the_right_dimension(): + """非方形画布:高度几何(脚线 / 占高)按高走,水平居中按宽走 —— 不能串轴。""" + out = align_bottom_center(_frames(), ref_height=300.0, cell=384, cell_h=512) + assert out[0].size == (384, 512) + height, foot, center = _subject(out[0]) + assert abs(height - 512 * FILL_H) <= 2, "主体占高必须按画布高算" + assert abs(foot - FOOT_LINE) <= 0.01, "脚线必须按画布高算" + assert abs(center - 0.5) <= 0.01, "水平居中必须按画布宽算" + + +def test_subject_fill_ratio_is_scale_invariant(): + """几何是"比例"不是"像素":换画布尺寸,主体占画布高的比例不变。 + + 这条是"母版入口预检与出帧共用同一套几何"的直接证据 —— 预检阈值 + (master_check.REJECT_ASPECT = 2*FILL_W/FILL_H)里没有 cell,本就与画布像素尺寸无关。 + """ + src = _frames() + ratios = [] + for cell in (128, 256, 512, 1024): + out = align_bottom_center(src, ref_height=300.0, cell=cell) + ratios.append(_subject(out[0])[0] / cell) + assert max(ratios) - min(ratios) < 0.01, f"占高比例应恒定,实测 {ratios}" + + +def test_width_fallback_uses_canvas_width_not_height(): + """宽度兜底(横向长条主体)要按画布**宽**收缩,否则宽画布上会白白缩小主体。""" + wide = [f.transpose(Image.ROTATE_90) for f in _frames(bh=300, bw=60)] + narrow = align_bottom_center(wide, cell=256, cell_h=256) + widened = align_bottom_center(wide, cell=512, cell_h=256) + # 画布变宽后,宽度兜底放松,主体应当更大(若按高算则两者相同) + assert _subject(widened[0])[0] > _subject(narrow[0])[0] + + +def test_non_positive_canvas_raises_instead_of_emitting_empty_image(): + """0 边长不静默出图:PIL 允许建 0×0,错产物要到落库/前端才暴露。""" + import pytest + + for kw in (dict(cell=0), dict(cell_h=0), dict(cell=-1)): + with pytest.raises(ValueError, match="画布尺寸"): + align_bottom_center(_frames(), **kw) + + +def test_all_transparent_frames_still_honour_requested_canvas(): + """全透明输入的兜底画布也要用请求的尺寸,不能退回 256 方形。""" + blank = [Image.new("RGBA", (64, 64), (0, 0, 0, 0)) for _ in range(3)] + out = align_bottom_center(blank, cell=320, cell_h=200) + assert [f.size for f in out] == [(320, 200)] * 3 diff --git a/backend/tests/test_pixelate.py b/backend/tests/test_pixelate.py new file mode 100644 index 00000000..11e5bdae --- /dev/null +++ b/backend/tests/test_pixelate.py @@ -0,0 +1,103 @@ +"""像素化后处理测试(纯 CV,无需联网 / API)。""" + +import numpy as np +from PIL import Image + +from windup_ai_engine.postprocess import ( + pixelate_frames, + sprite_sheet, + to_pixel_art, +) + + +def _synthetic_char(size=256, box=(80, 40, 176, 220)) -> Image.Image: + """透明底上画一个不透明矩形"角色",四周留透明边。""" + img = Image.new("RGBA", (size, size), (0, 0, 0, 0)) + arr = np.asarray(img).copy() + x0, y0, x1, y1 = box + arr[y0:y1, x0:x1] = (200, 60, 60, 255) + # 加一点颜色变化,让色板量化有意义 + arr[y0:y1, x0 : (x0 + x1) // 2] = (60, 120, 200, 255) + return Image.fromarray(arr, "RGBA") + + +def test_to_pixel_art_targets_height_and_keeps_ratio(): + src = _synthetic_char() # 主体 96x180 + out = to_pixel_art(src, target_h=60, palette_size=16) + assert out.height == 60 + # 主体宽高比 96/180 → 目标宽 ≈ 60*96/180 = 32 + assert abs(out.width - 32) <= 1 + assert out.mode == "RGBA" + + +def test_to_pixel_art_crops_to_alpha_bbox(): + """输出应裁到主体包围盒:透明边被切掉,首列即主体。""" + out = to_pixel_art(_synthetic_char(), target_h=90, palette_size=16) + alpha = np.asarray(out)[:, :, 3] + assert alpha.max() == 255 # 有实心主体 + # 顶行与左列应落在主体上(已裁边),而非全透明 + assert alpha[0, :].max() > 0 + assert alpha[:, 0].max() > 0 + + +def test_to_pixel_art_reduces_palette(): + out = to_pixel_art(_synthetic_char(), target_h=80, palette_size=8) + rgb = np.asarray(out.convert("RGB")).reshape(-1, 3) + colors = np.unique(rgb, axis=0) + assert len(colors) <= 8 + + +def test_pixelate_frames_uniform_height_packs_to_sheet(): + frames = pixelate_frames([_synthetic_char() for _ in range(4)], target_h=48, palette_size=16) + assert all(f.height == 48 for f in frames) + sheet = sprite_sheet(frames) + assert sheet.height == 48 + assert sheet.width == sum(f.width for f in frames) + + +def test_to_pixel_art_rejects_bad_height(): + import pytest + + with pytest.raises(ValueError): + to_pixel_art(_synthetic_char(), target_h=0) + + +def _pixel_art(block=8, logical_h=20, bg=(255, 255, 255)) -> Image.Image: + """合成像素画:每个逻辑像素放大成 block×block 方块,白底(模拟母版)。""" + colors = [(200, 60, 60), (60, 120, 200), (40, 160, 90)] + small = np.full((logical_h, logical_h // 2, 3), bg, dtype=np.uint8) + for y in range(4, logical_h - 4): + for x in range(2, logical_h // 2 - 2): + small[y, x] = colors[(x + y) % len(colors)] + img = Image.fromarray(small, "RGB").resize( + (small.shape[1] * block, logical_h * block), Image.NEAREST + ) + return img.convert("RGBA") + + +def test_detect_pixel_size_finds_block(): + from windup_ai_engine.postprocess import detect_pixel_size + + assert detect_pixel_size(_pixel_art(block=8)) == 8 + assert detect_pixel_size(_pixel_art(block=12)) == 12 + + +def test_master_pixel_spec_gives_logical_height_and_palette(): + from windup_ai_engine.postprocess import master_pixel_spec + + logical_h, palette = master_pixel_spec(_pixel_art(block=8, logical_h=20)) + assert 10 <= logical_h <= 14 # 主体(去掉白边)约 12 个逻辑像素高 + assert 2 <= len(palette) <= 32 + # 色板不应被白底/抗锯齿近白色占据 + assert not (palette.astype(int).sum(axis=1) > 700).all() + + +def test_palette_lock_restricts_output_colors(): + """锁色板后,输出颜色必须全部来自给定色板(用于消掉压缩灰颗粒)。""" + palette = np.array([[200, 60, 60], [60, 120, 200]], dtype=np.uint8) + noisy = _synthetic_char() + out = to_pixel_art(noisy, target_h=24, palette=palette) + rgb = np.asarray(out.convert("RGB")).reshape(-1, 3) + used = np.unique(rgb, axis=0) + for c in used: + assert (c == palette).all(axis=1).any(), f"{c} 不在色板内" diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py new file mode 100644 index 00000000..907094f9 --- /dev/null +++ b/backend/tests/test_sufy_video_download.py @@ -0,0 +1,596 @@ +"""视频成品下载的凭证边界、重试与完整性校验(不联网:用 httpx MockTransport)。 + +两个回归对象: + +1. 2026-08-05 实测两次连续复现:视频已生成、费用已产生,却因为读 body 时断了一次连接 + 就整单丢弃。见 ``providers.sufy._download`` 的 docstring。 +2. 2026-08-10 机器审(PR #179 P1):成品 URL 是网关返回的绝对地址,复用带 Authorization + 的 client 去下载 = 把 API key 发给了 CDN(或网关返回的任意地址)。 + 见 ``providers.sufy._download_request`` 的 docstring。 +""" + +import json + +import httpx +import pytest + +from windup_framework.providers.sufy import ( + IncompleteDownloadError, + UnsafeDownloadUrlError, + _download, +) + +VIDEO = b"\x00\x01mp4-bytes" * 64 +GATEWAY = "https://gw.invalid/v1" + + +def _client(handler) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def _authed_client(handler, base_url: str = GATEWAY) -> httpx.Client: + """带凭证的网关 client —— provider 真正持有的就是这种(Authorization + cookie jar)。""" + return httpx.Client( + transport=httpx.MockTransport(handler), + base_url=base_url, + headers={"Authorization": "Key secret-api-key"}, + cookies={"session": "s3cr3t"}, + ) + + +def test_retries_after_peer_closed_connection(monkeypatch): + """第一次断连、第二次成功 —— 原实现在这里会整单丢弃。""" + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + raise httpx.RemoteProtocolError( + "peer closed connection without sending complete message body", request=request + ) + return httpx.Response(200, content=VIDEO) + + with _client(handler) as client: + assert _download(client, "https://example.invalid/v.mp4") == VIDEO + assert calls["n"] == 2 + + +def test_rejects_truncated_body_that_does_not_raise(monkeypatch): + """服务端声明的长度与实收不符时必须失败,而不是把坏视频往下游送。 + + 截断不一定抛异常。放过去的话,坏视频要到出帧环节才暴露成"解码失败", + 很难回溯到下载这一步。 + """ + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + + def handler(request: httpx.Request) -> httpx.Response: + # 只回一半 body,但 Content-Length 仍声明全长 + return httpx.Response( + 200, content=VIDEO[: len(VIDEO) // 2], headers={"content-length": str(len(VIDEO))} + ) + + with _client(handler) as client, pytest.raises(RuntimeError, match="已重试 3 次"): + _download(client, "https://example.invalid/v.mp4") + + +def test_accepts_chunked_response_without_content_length(monkeypatch): + """分块传输没有 Content-Length,此时跳过校验而不是误判为不完整。""" + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, stream=httpx.ByteStream(VIDEO)) + + with _client(handler) as client: + assert _download(client, "https://example.invalid/v.mp4") == VIDEO + + +def test_gives_up_after_three_tries_and_reports_the_last_cause(monkeypatch): + """一直断连时要显式失败,并把最后一次的真实原因带出来。""" + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + raise httpx.ConnectError("connection reset", request=request) + + with _client(handler) as client, pytest.raises(RuntimeError, match="connection reset"): + _download(client, "https://example.invalid/v.mp4") + assert calls["n"] == 3 + + +def test_incomplete_download_error_is_a_runtime_error(): + """调用方按 RuntimeError 兜底即可,不必单独 import 这个子类。""" + assert issubclass(IncompleteDownloadError, RuntimeError) + + +# ── 凭证边界:成品 URL 是网关给的外部地址,不能带着 API key 去取 ────────────── + + +def test_cross_origin_download_does_not_leak_the_api_key(monkeypatch): + """跨源下载必须摘掉 client 级凭证。 + + 这是 PR #179 P1 的直接回归:httpx 只在跨源**重定向**时自动摘 Authorization, + 对一开始就跨源的直连请求会原样带上 —— 于是 CDN 域名收到了 API key。 + """ + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + seen: dict[str, str | None] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["authorization"] = request.headers.get("authorization") + seen["cookie"] = request.headers.get("cookie") + return httpx.Response(200, content=VIDEO) + + with _authed_client(handler) as client: + assert _download(client, "https://cdn.invalid/out.mp4") == VIDEO + + assert seen["authorization"] is None, "API key 被发给了 CDN" + assert seen["cookie"] is None, "会话 cookie 被发给了 CDN" + + +def test_same_origin_download_keeps_the_gateway_credential(monkeypatch): + """同源(网关自己签发的下载链接)必须保留凭证,否则那条路径就是 401。 + + 一律摘头会把这个功能弄坏,所以判据是目标地址,不是"下载一律不带凭证"。 + """ + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + seen: list[str | None] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.headers.get("authorization")) + return httpx.Response(200, content=VIDEO) + + with _authed_client(handler) as client: + # 第二个地址显式写出默认端口 443。httpx 0.28 会把默认端口归一化掉(URL.port -> None), + # 所以这条今天走不到"补默认端口"那行;留着是钉住这个前提 —— httpx 哪天不再归一化, + # 少了默认端口补齐就会把它误判成跨源、把凭证摘掉,这条会先叫。 + assert _download(client, "https://gw.invalid/files/out.mp4") == VIDEO + assert _download(client, "https://gw.invalid:443/files/out.mp4") == VIDEO + + assert seen == ["Key secret-api-key", "Key secret-api-key"] + + +def test_downgrade_to_plain_http_is_treated_as_cross_origin(monkeypatch): + """同 host 但 scheme 从 https 掉到 http —— 也要摘凭证。 + + 默认端口被 httpx 归一化成 None,host 又相同,所以同源判定里**少比一个 scheme** + 就会把它当自己人,于是 API key 走明文 HTTP 发出去。httpx 自己在重定向那侧也是 + 单独处理 http/https 的(``_is_https_redirect``),方向只允许 http→https,不允许反过来。 + """ + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + seen: dict[str, str | None] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["authorization"] = request.headers.get("authorization") + return httpx.Response(200, content=VIDEO) + + with _authed_client(handler) as client: + assert _download(client, "http://gw.invalid/files/out.mp4") == VIDEO + assert seen["authorization"] is None, "API key 走明文 HTTP 发了出去" + + # 再来一格显式非默认端口:两边端口都是 8443,"补默认端口"那行判不出差别, + # 只有 scheme 比较能拦住。少了这一格,scheme 比较会显得可以删(实际不行)。 + with _authed_client(handler, base_url="https://gw.invalid:8443/v1") as client: + assert _download(client, "http://gw.invalid:8443/files/out.mp4") == VIDEO + assert seen["authorization"] is None, "非默认端口上的 https->http 降级没拦住" + + +def test_relative_result_path_stays_authenticated(monkeypatch): + """网关返回相对路径时,它解析到网关自己身上,凭证照带。""" + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + seen: dict[str, str | None] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["url"] = str(request.url) + seen["authorization"] = request.headers.get("authorization") + return httpx.Response(200, content=VIDEO) + + with _authed_client(handler) as client: + assert _download(client, "files/out.mp4") == VIDEO + + assert seen["url"] == "https://gw.invalid/v1/files/out.mp4" + assert seen["authorization"] == "Key secret-api-key" + + +def test_non_http_result_url_is_refused_before_any_request_goes_out(monkeypatch): + """协议不是 http(s) 就不发请求 —— 地址不对要立刻炸,不是重试三次后报传输错。 + + 注意 httpx 的边界:只有**带 host** 的绝对地址才保留原 scheme(``ftp://cdn/...``); + ``file:///etc/passwd`` 这种没有 host 的会被 httpx 当相对地址并入 base_url, + 结果是一个打到网关的 404,不经过这个分支。 + """ + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + + def handler(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"不该发出任何请求: {request.url}") + + for url in ("ftp://cdn.invalid/out.mp4", "file://cdn.invalid/out.mp4"): + with _authed_client(handler) as client: + with pytest.raises(UnsafeDownloadUrlError, match="http"): + _download(client, url) + + +# ── 文生图 provider（2026-08-10 实现；此前 gen_image 必抛错而端点可达）────────── + + +def _img_payload(b64: str) -> dict: + """模型把图放在 message.content 里，不同网关包裹层级不同。""" + return {"choices": [{"message": {"content": f"data:image/png;base64,{b64}"}}]} + + +def _big_b64(n: int = 6000) -> str: + import base64 + return base64.b64encode(b"\x89PNG" + b"\x00" * n).decode() + + +def _image_provider(handler): + import httpx + + from windup_framework.config.provider import AIProviderSettings + from windup_framework.providers.sufy import SufyImageProvider + + p = SufyImageProvider( + config=AIProviderSettings(base_url="https://gw.example.com/v1", api_key="k"), + ) + client = httpx.Client( + base_url="https://gw.example.com/v1", + headers={"Authorization": "Bearer k"}, + transport=httpx.MockTransport(handler), + ) + p._client = lambda: client + return p + + +def test_gen_image_returns_the_decoded_png(): + """端点可达而 provider 必抛错 = 每个图像任务稳定 FAILED。实现后必须真能出图。""" + def h(request): + import httpx + return httpx.Response(200, json=_img_payload(_big_b64())) + + data = _image_provider(h).gen_image("a knight", []) + assert data.startswith(b"\x89PNG") and len(data) > 5000 + + +def test_reference_images_are_sent_as_data_uris(): + """参考图走 content 数组里的 image_url，不是 multipart、不是单独字段。""" + import json as _json + + seen: dict = {} + + def h(request): + import httpx + seen["body"] = _json.loads(request.content) + return httpx.Response(200, json=_img_payload(_big_b64())) + + _image_provider(h).gen_image("x", [b"\x89PNGref"]) + content = seen["body"]["messages"][0]["content"] + kinds = [c["type"] for c in content] + assert kinds == ["text", "image_url"] + assert content[1]["image_url"]["url"].startswith("data:image/png;base64,") + + +def test_response_without_an_image_is_retried_then_raises(): + """模型偶发返回一条不含图的正常响应。重试后仍拿不到必须抛，不能返回空 bytes—— + 上游会把返回值直接上传对象存储并写进任务结果，0 字节的"成功"就是用户看到的裂图。""" + import pytest + + calls = {"n": 0} + + def h(request): + import httpx + calls["n"] += 1 + return httpx.Response(200, json={"choices": [{"message": {"content": "抱歉"}}]}) + + with pytest.raises(RuntimeError, match="未取得有效图"): + _image_provider(h).gen_image("x", []) + assert calls["n"] == 3, "应重试到上限而不是一次就放弃" + + +def test_undersized_image_is_rejected_not_returned(): + """响应里可能带一个几十字节的占位串，当图存下去就是打不开的文件。""" + import base64 + + import pytest + + tiny = base64.b64encode(b"\x89PNG" + b"\x00" * 200).decode() + + def h(request): + import httpx + return httpx.Response(200, json=_img_payload(tiny)) + + with pytest.raises(RuntimeError, match="字节"): + _image_provider(h).gen_image("x", []) + + +def test_first_successful_attempt_stops_retrying(): + calls = {"n": 0} + + def h(request): + import httpx + calls["n"] += 1 + if calls["n"] == 1: + return httpx.Response(200, json={"choices": [{"message": {"content": "空"}}]}) + return httpx.Response(200, json=_img_payload(_big_b64())) + + assert _image_provider(h).gen_image("x", []) + assert calls["n"] == 2 + + +def test_image_client_retries_connection_failures(): + """本机走代理时建连抖动常见；已跑通的管线实现靠一层网络重试扛住。 + + 只断言"配了连接重试"这个结构 —— 真去模拟 SSL 握手失败需要一个假 TCP 端点， + 那验的是 httpx 而不是我们的代码。 + """ + from windup_framework.providers.sufy import _CONNECT_RETRIES, SufyImageProvider + + assert _CONNECT_RETRIES >= 1 + client = SufyImageProvider()._client() + try: + assert client._transport._pool._retries == _CONNECT_RETRIES + finally: + client.close() + + +def test_request_path_comes_from_config_not_a_literal(): + """路径用配置里的 chat_completions_path —— 它此前零消费方,正是今天在删的那类字段。""" + + seen: dict = {} + + def h(request): + import httpx + seen["path"] = request.url.path + return httpx.Response(200, json=_img_payload(_big_b64())) + + p = _image_provider(h) + p._cfg = p._cfg.model_copy(update={"chat_completions_path": "/v9/custom-chat"}) + p.gen_image("x", []) + assert seen["path"].endswith("/v9/custom-chat"), seen["path"] + + +@pytest.mark.parametrize("code", [400, 404]) +def test_model_missing_from_the_gateway_catalogue_says_so(code): + """同一把 key 下不同网关的模型目录不一样(实测:一个 73 个模型零图像模型、 + 另一个 134 个含默认模型)。配错 AI_BASE_URL 时错误必须指向配置,不能只是裸 404。 + """ + def h(request): + import httpx + return httpx.Response(code, text='{"error":{"message":"model not found"}}') + + with pytest.raises(RuntimeError, match=r"/models"): + _image_provider(h).gen_image("x", []) + + +# ── 模型型号可配置（2026-08-11 人工评审：providers 层硬编码太多）─────────────── + + +def _cfg(**kw): + from windup_framework.config.provider import AIProviderSettings + + return AIProviderSettings(base_url="https://gw.example.com/v1", api_key="k", **kw) + + +@pytest.mark.parametrize(("cls_name", "field", "value"), [ + ("SufyVideoProvider", "video_model", "kling-v9-test"), + ("SufyImageProvider", "image_model", "gemini-9-flash-image"), +]) +def test_each_provider_reads_its_own_model_field(cls_name, field, value): + """三条能力同时在用不同模型，所以是三个独立字段而不是共用一个 ``model``。 + + 共用一个的后果是换其中一条把另外两条也换了 —— 这条用例把"各读各的"钉住： + 只设自己那个字段，另外两个保持默认，断言取到的是自己的。 + """ + import windup_framework.providers.sufy as S + + cls = getattr(S, cls_name) + assert cls(config=_cfg(**{field: value}))._model == value + + +def test_explicit_model_argument_still_wins_over_config(): + """显式传参优先于配置 —— A/B 对比时不必改环境变量。""" + from windup_framework.providers.sufy import SufyImageProvider + + p = SufyImageProvider(config=_cfg(image_model="from-config"), model="from-arg") + assert p._model == "from-arg" + + +def test_request_shape_is_not_configurable(): + """**只有型号可配，请求形状不可配。** + + （FAL 队列面已随「从未真实调用过」一并移除，故这里只剩两个型号字段。） + + 哪个模型吃 image_list、FAL 队列路径长什么样，是该模型的 API 事实而非运行参数。 + 放进配置会把"填错了会怎样"从部署期推到运行期：字段塞错不会立刻报错，任务照常 + queued，直到生成阶段才 failed，而费用可能已经产生（2026-07-29 实测）。 + + 故断言配置类**没有**这类字段 —— 将来有人想加会先撞到这条用例和它的理由。 + """ + from windup_framework.config.provider import AIProviderSettings + + fields = set(AIProviderSettings.model_fields) + for banned in ("image_list_models", "fal_endpoints", "first_frame_field"): + assert banned not in fields, f"{banned} 不该进配置，见本用例 docstring" + assert {"video_model", "image_model"} <= fields + + +# ── i2v 主流程（付费路径，此前零覆盖）───────────────────────────────────────── + + +def _jpeg_first_frame(w: int = 200, h: int = 300) -> bytes: + """一张竖长的图，用来验首帧被按目标画布补边而不是拉伸。""" + import io as _io + + from PIL import Image as _Image + + buf = _io.BytesIO() + _Image.new("RGB", (w, h), (40, 80, 160)).save(buf, "PNG") + return buf.getvalue() + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch): + """轮询里的 time.sleep 打桩 —— 用例不该真等。""" + import windup_framework.providers.sufy as _S + + monkeypatch.setattr(_S.time, "sleep", lambda *_: None) + + +def _video_provider(handler, **kw): + import httpx as _httpx + + from windup_framework.config.provider import AIProviderSettings + from windup_framework.providers.sufy import SufyVideoProvider + + p = SufyVideoProvider( + config=AIProviderSettings(base_url="https://gw.example.com/v1", api_key="k"), + # 轮询预算 = max_min * 60 // poll。poll 取大值让预算只有几次， + # 再把 time.sleep 打桩掉，用例就既快又不空转（第一版 poll=0.001 配 + # max_min=1 会真轮询 6 万次，单文件跑了 96 秒）。 + poll_interval=30.0, + **kw, + ) + client = _httpx.Client( + base_url="https://gw.example.com/v1", + headers={"Authorization": "Bearer k"}, + transport=_httpx.MockTransport(handler), + ) + p._client = lambda: client + return p + + +def _i2v_handler(seen: dict, *, statuses=("completed",), video=b"MP4DATA" * 200): + """提交 → 轮询 → 下载 三段式的假网关。""" + import httpx as _httpx + + calls = {"n": 0} + + def h(request): + path = request.url.path + if request.method == "POST" and path.endswith("/videos"): + seen["body"] = json.loads(request.content) + return _httpx.Response(200, json={"id": "job-1"}) + if request.method == "GET" and "/videos/" in path: + i = min(calls["n"], len(statuses) - 1) + calls["n"] += 1 + st = statuses[i] + if st == "completed": + return _httpx.Response(200, json={ + "status": "completed", + "task_result": {"videos": [{"url": "https://gw.example.com/out.mp4"}]}, + }) + return _httpx.Response(200, json={"status": st, "error": "boom"}) + seen["download_headers"] = dict(request.headers) + return _httpx.Response(200, content=video, + headers={"Content-Length": str(len(video))}) + + return h + + +def test_i2v_submits_polls_and_downloads(): + """一条完整的付费路径：提交拿 job id → 轮询到 completed → 下载 mp4。""" + seen: dict = {} + data = _video_provider(_i2v_handler(seen)).i2v(_jpeg_first_frame(), "walk right") + assert data.startswith(b"MP4DATA") + body = seen["body"] + assert body["prompt"] == "walk right" + assert body["seconds"] == "5" and isinstance(body["seconds"], str), "seconds 必须是字符串" + assert body["mode"] == "std" + + +def test_first_frame_goes_as_a_jpeg_data_uri(): + """PNG base64 会让任务 status=failed（VENDOR_FAILED，2026-07-22 实测，33s fail-fast）。 + 首帧必须转 JPEG —— 这条错在提交后才报，本地看不出来。 + """ + seen: dict = {} + _video_provider(_i2v_handler(seen)).i2v(_jpeg_first_frame(), "x") + uri = seen["body"]["input_reference"] + assert uri.startswith("data:image/jpeg;base64,"), uri[:40] + + import base64 as _b64 + import io as _io + + from PIL import Image as _Image + + im = _Image.open(_io.BytesIO(_b64.b64decode(uri.split(",", 1)[1]))) + assert im.format == "JPEG" + + +def test_first_frame_is_padded_to_the_target_canvas_not_stretched(): + """按目标画布补边、不拉伸：拉伸会让角色比例变形，而母版比例是角色一致性的一部分。""" + import base64 as _b64 + import io as _io + + from PIL import Image as _Image + + # 源图放一个偏心的亮块：拉伸会把它拉宽，补边会保持它的宽高比。 + # 只看"对称两点颜色相同"是无效判据 —— 纯色图拉伸后照样相同 + # （2026-08-11 变异测试逮到第一版正是如此，M3 存活）。 + buf = _io.BytesIO() + src = _Image.new("RGB", (200, 300), (40, 80, 160)) + src.paste((250, 250, 250), (80, 100, 120, 140)) # 40x40 的方块 + src.save(buf, "PNG") + + seen: dict = {} + _video_provider(_i2v_handler(seen)).i2v(buf.getvalue(), "x", size="1280x720") + im = _Image.open(_io.BytesIO(_b64.b64decode(seen["body"]["input_reference"].split(",", 1)[1]))) + assert im.size == (1280, 720), "首帧应铺满目标画布" + + # 量那个方块在成品里的宽高比。补边：源 40x40 等比缩放后仍是 1:1。 + # 拉伸：横向被拉 1280/200=6.4 倍、纵向 720/300=2.4 倍，比例变成 ~2.67:1。 + import numpy as _np + + a = _np.asarray(im.convert("L")) + ys, xs = _np.where(a > 200) + ratio = (xs.max() - xs.min() + 1) / (ys.max() - ys.min() + 1) + assert 0.8 < ratio < 1.25, f"方块宽高比 {ratio:.2f}，说明被拉伸了（补边应≈1.0）" + + +@pytest.mark.parametrize("bad", ["failed", "cancelled"]) +def test_terminal_failure_raises_instead_of_polling_to_timeout(bad): + """网关报 failed/cancelled 要立刻抛，别把剩下的轮询次数耗完 —— 钱已经花了， + 尽快把原因暴露给上层比多等几分钟有用。 + """ + with pytest.raises(RuntimeError, match=bad): + _video_provider(_i2v_handler({}, statuses=(bad,))).i2v(_jpeg_first_frame(), "x") + + +def test_never_completing_job_raises_after_the_poll_budget(): + """轮询预算用尽仍未 completed → 抛错，不返回空 bytes。 + + 返回空 bytes 的话上游会把它当成一段视频送进抽帧，报"视频无可解码帧"， + 真正的原因（超时）就被埋掉了。 + """ + p = _video_provider(_i2v_handler({}, statuses=("in_progress",)), max_min=1) # 预算 2 次 + with pytest.raises(RuntimeError, match="未取得视频 URL"): + p.i2v(_jpeg_first_frame(), "x") + + +def test_image_list_models_use_a_different_first_frame_field(): + """字段按模型选。塞错字段不会立刻报错 —— 任务 status=queued 正常返回， + 直到生成阶段才 failed "model is not supported"，而费用可能已经产生 + （2026-07-29 实测）。 + """ + from windup_framework.providers.sufy import _IMAGE_LIST_MODELS + + seen: dict = {} + p = _video_provider(_i2v_handler(seen), model=_IMAGE_LIST_MODELS[0], mode="pro") + p.i2v(_jpeg_first_frame(), "x") + assert "image_list" in seen["body"] and "input_reference" not in seen["body"] + assert not seen["body"]["image_list"][0]["image"].startswith("data:"), \ + "image_list 要裸 base64，不带 data URI 前缀" + + +def test_non_positive_poll_interval_is_rejected_at_construction(): + """轮询间隔 <= 0 在构造时就拒。 + + 此前会活到 i2v 里 `max_min * 60 // poll` 那一步除零 —— 报 ZeroDivisionError， + 读的人完全看不出是配错了参数（2026-08-11 补 i2v 主流程测试时逮到）。 + 0 的语义本身也不成立：那是忙等，会把网关打满。 + """ + from windup_framework.config.provider import AIProviderSettings + from windup_framework.providers.sufy import SufyVideoProvider + + cfg = AIProviderSettings(base_url="https://gw.example.com/v1", api_key="k") + for bad in (0, -1, -0.5): + with pytest.raises(ValueError, match="poll_interval"): + SufyVideoProvider(config=cfg, poll_interval=bad) diff --git a/backend/uv.lock b/backend/uv.lock index bfcbe78f..5e1a6cfd 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -49,6 +49,32 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494" }, ] +[[package]] +name = "av" +version = "18.0.0" +source = { registry = "https://mirrors.aliyun.com/pypi/simple/" } +sdist = { url = "https://mirrors.aliyun.com/pypi/packages/ae/a4/570a5a35c8638aba01e739925846c35fdd6b0756a15526766d0a4dd3b7df/av-18.0.0.tar.gz", hash = "sha256:4ef7e72c3d3a872584a1215173b16e0226811037f40dcdbf75992631098df1ba" } +wheels = [ + { url = "https://mirrors.aliyun.com/pypi/packages/18/4a/9e3463df030e063d757fa12f0f39be6541b45b06b5bad48c2ce361b924bf/av-18.0.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:149289d40e732a6e49c9530bc245b49d9964cfd1c8c9e06778703b7d5bba6b25" }, + { url = "https://mirrors.aliyun.com/pypi/packages/77/b3/2576a44b4f39c7462ced4c17fec04c756f7b0f3c5cb940d124173e417d6a/av-18.0.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:35274c20d2ad3b4774fe632bcef2e34af79858ddf899352339cc3babbc13a484" }, + { url = "https://mirrors.aliyun.com/pypi/packages/84/74/6732f17b96dc23fd23b876b2805435855abdc8a3b397142be4e581165de8/av-18.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4d683b7747a0ba9222b8a5f81e41db5f796e7f64473454ec4fe2548e083c2fa0" }, + { url = "https://mirrors.aliyun.com/pypi/packages/6d/b9/7708c43fed7ae28b4a1bad060b4221e3334cd827cec24f7165902a6ac1f4/av-18.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ae56b40b6f8b067a8ad2dac664fbfbabac7f7a55b9a7bb031eb99289252bc017" }, + { url = "https://mirrors.aliyun.com/pypi/packages/5a/94/eba99691d184f6a395a242d54dc370e2fd2265e95bbc98e2963a0fdbdd6c/av-18.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ea2e8ebbce521f21b55df9400e00d721623c9020ef158f5a188a96130be0743f" }, + { url = "https://mirrors.aliyun.com/pypi/packages/c9/cf/0d7aee07fe16aa9ffdf96043c14bed5485a52c0dea4259de87aa306ecab4/av-18.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef96dabb3e50dac249913145dff5424b302b257fd95dcb64be3c7b7a8aef16d1" }, + { url = "https://mirrors.aliyun.com/pypi/packages/76/92/810da80b12680d4c4fe235bd1b4003289be9213ac7f114b77b8ecf0e3b3e/av-18.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0f65518a184613e41536f29e8758c8e3d8293e46bf5bef108f04f925bbfa3f44" }, + { url = "https://mirrors.aliyun.com/pypi/packages/11/85/0f121ff43dc5a70696676c98a8f1674e2fa787614c2abaacb15fa1a9bc99/av-18.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:aaf4d354d2beaa6651e4f92e54409a578bde64f79c0beef9a30b388d06f7c629" }, + { url = "https://mirrors.aliyun.com/pypi/packages/8b/f6/2509754d4d2356abc6fc0ea3d57c12ade29bac23a1fb7fc215a53ca518fb/av-18.0.0-cp311-abi3-win_arm64.whl", hash = "sha256:adac2b3833b6cb9bd6cb52664a522b94db453615b3675b1dbb26e13fe1c80da6" }, + { url = "https://mirrors.aliyun.com/pypi/packages/e2/25/4ee23a7f1609adf9b2f140c7a8ffade64a1449d89ab431d922a809eebf19/av-18.0.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:88dd8e35e9242662b409a6a05fd24a6775d949eb05da0ba31cab4f250eacbab5" }, + { url = "https://mirrors.aliyun.com/pypi/packages/f1/f0/b9f8363d07aa4521913e483f6a30c7c164973ef01de62769bf9b97049cd8/av-18.0.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f8f454349c402e2c8d6fa80b54eb2a3f86c00f414d2b399f01ae6dab075c6fd8" }, + { url = "https://mirrors.aliyun.com/pypi/packages/c3/e5/69397019aed280a72a43e97a252dee4295df1a9e608848452e5300ec4dab/av-18.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:88ce194c2201c6a6d40336adee8a5ddde46ed743eacb500e3ae9368d1c6d889e" }, + { url = "https://mirrors.aliyun.com/pypi/packages/37/3a/1614d74f0d676ea6745eb59553c9ad01ca25db523cba808d522e838f4f5b/av-18.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:aa15e567a018cc94a26b0ab45da676dee70c4146ace6e92e47d30cc9689cbfbe" }, + { url = "https://mirrors.aliyun.com/pypi/packages/6b/3c/5f54710d69b0ea93634134f92b49c7a2a7fd27da5486a8a7e6251ac1cfb4/av-18.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:613153e48cefc91700746dde0ad0282d4677b194cba22cc771de14c78411cf8b" }, + { url = "https://mirrors.aliyun.com/pypi/packages/26/92/8293e6a267e0591b543abd96ae01e7e8ed228509bdb4e4644a8a8395d90f/av-18.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:30404f53ca1ea7f350ac86ff22a2c04f903014758e9b33f398c5a62de34bd84f" }, + { url = "https://mirrors.aliyun.com/pypi/packages/10/0c/38ed7601277ae57dfe857d040be4762530fd728efff45c2fb8f035fef96a/av-18.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6882a48f7aec2863c96cddee3256ff2da98f7fb6cbed83cee9d7e70a8f186a6b" }, + { url = "https://mirrors.aliyun.com/pypi/packages/c8/95/0636ca04d5d89d01c49bd366d2b660cc85d1f8117c476b2be62eb0c70855/av-18.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:55a646e9afce9fdc5de5224205a8a12c7ed1ba9803145dcc876c40bfc03a109b" }, + { url = "https://mirrors.aliyun.com/pypi/packages/01/20/1e24450ea981c44ed328691496fd2774dfa9fa3c3b00fd07f72fd5614abe/av-18.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:96f594ff506a09475e5549359352332049a25d37a08f00b4623f7f6e92e45b9c" }, +] + [[package]] name = "bcrypt" version = "5.0.0" @@ -585,6 +611,19 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2" }, ] +[[package]] +name = "imageio" +version = "2.37.4" +source = { registry = "https://mirrors.aliyun.com/pypi/simple/" } +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, +] +sdist = { url = "https://mirrors.aliyun.com/pypi/packages/48/62/aa770a9307508d2a2a2c62d536a49347bffe9e55322db27838d3c93d0b07/imageio-2.37.4.tar.gz", hash = "sha256:e45cbc5e83502047fb138f7f585f7f105a136a57eea5f4b3cfc6ce1b52720bd3" } +wheels = [ + { url = "https://mirrors.aliyun.com/pypi/packages/3e/2d/ca050652104bab2cf55e569db2a178b1b61cb041fef28307f2db383f6d9f/imageio-2.37.4-py3-none-any.whl", hash = "sha256:1ab2e22c8debf700f24c3ac43e8f95f3b3a8110c83b93411e97b4b0b2cd1c7e6" }, +] + [[package]] name = "import-linter" version = "2.13" @@ -938,7 +977,7 @@ wheels = [ [[package]] name = "openai" -version = "2.50.0" +version = "2.53.0" source = { registry = "https://mirrors.aliyun.com/pypi/simple/" } dependencies = [ { name = "anyio" }, @@ -950,9 +989,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://mirrors.aliyun.com/pypi/packages/d8/f5/e7735f2af272ee179a287911a698b3cbdb59d7a4ac4874571363adf1e4de/openai-2.50.0.tar.gz", hash = "sha256:5128f7caf4a6b01aefd6e7e93efe170a2c3427b8de286b9af5cdff3aa47e02c8" } +sdist = { url = "https://mirrors.aliyun.com/pypi/packages/ef/cf/36e3e7235fdf6d125c052acc0970924611b17a20a4fe580596faf4566a65/openai-2.53.0.tar.gz", hash = "sha256:baf5802ad08980e1d9d561e1b996e800c8bcd14af5847c6d0e7a5cc59e4d4116" } wheels = [ - { url = "https://mirrors.aliyun.com/pypi/packages/00/ca/db315b3bb748c26c644a3f85b7d509e774354d6518d47080b1446005ee41/openai-2.50.0-py3-none-any.whl", hash = "sha256:90bdddcc5a2fa529b350fac9c5780d87e5c361dcc6090ab57b0d470b0d7af7fa" }, + { url = "https://mirrors.aliyun.com/pypi/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618" }, ] [[package]] @@ -2065,6 +2104,8 @@ name = "windup-ai-engine" version = "0.1.0" source = { editable = "packages/ai_engine" } dependencies = [ + { name = "av" }, + { name = "imageio" }, { name = "langchain-core" }, { name = "langgraph" }, { name = "numpy" }, @@ -2075,6 +2116,8 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "av", specifier = ">=14.0" }, + { name = "imageio", specifier = ">=2.36" }, { name = "langchain-core", specifier = ">=0.3" }, { name = "langgraph", specifier = ">=0.2" }, { name = "numpy", specifier = ">=1.26" },