diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 5d4ad7fbb..75274b9f1 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -1797,7 +1797,7 @@ struct LLMEmbedder : public Conditioner { arch = LLM::LLMArch::GPT_OSS_20B; } else if (sd_version_is_pid(version)) { arch = LLM::LLMArch::GEMMA2_2B; - } else if (sd_version_is_ideogram4(version) || sd_version_is_boogu_image(version) || sd_version_is_sefi_image(version) || sd_version_is_krea2(version)) { + } else if (sd_version_is_lingbot_video(version) || sd_version_is_ideogram4(version) || sd_version_is_boogu_image(version) || sd_version_is_sefi_image(version) || sd_version_is_krea2(version)) { arch = LLM::LLMArch::QWEN3_VL; } else if (sd_version_is_z_image(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) { arch = LLM::LLMArch::QWEN3; @@ -2009,7 +2009,84 @@ struct LLMEmbedder : public Conditioner { int64_t t0 = ggml_time_ms(); - if (sd_version_is_qwen_image(version)) { + if (sd_version_is_lingbot_video(version)) { + const int pad_token = 151643; + const std::string prompt_prefix = + "<|im_start|>system\nGiven a user input that may include a text prompt alone, " + "a text prompt with an image reference, or a text prompt with a video reference " + "or a video reference alone, generate an \"Enhanced prompt\" that provides detailed " + "visual descriptions suitable for video generation. Evaluate the level of detail " + "in the user's input: if it is simple, enrich it by adding specifics about colors, " + "shapes, sizes, textures, lighting, motion dynamics, camera movement, temporal " + "progression, and spatial relationships to create vivid, concrete, and temporally " + "coherent scenes to create vivid and concrete scenes. Please generate only the " + "enhanced description for the prompt below and avoid including any additional " + "commentary or evaluations:<|im_end|>\n<|im_start|>user\n"; + + auto prefix_tokens = tokenizer->encode(prompt_prefix, nullptr); + prompt_template_encode_start_idx = 0; + for (int token : prefix_tokens) { + if (token != pad_token) { + prompt_template_encode_start_idx++; + } + } + LOG_DEBUG("prompt_template_encode_start_idx %d", prompt_template_encode_start_idx); + + prompt = prompt_prefix; + if (llm->enable_vision && conditioner_params.ref_images != nullptr && !conditioner_params.ref_images->empty()) { + LOG_INFO("LingBotVideoI2VPipeline"); + const std::string placeholder = "<|image_pad|>"; + std::string img_prompt; + + for (int i = 0; i < conditioner_params.ref_images->size(); i++) { + const auto& image = (*conditioner_params.ref_images)[i]; + double factor = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size; + int height = static_cast(image.shape()[1]); + int width = static_cast(image.shape()[0]); + int min_pixels = static_cast(4 * factor * factor); + int max_pixels = static_cast(16384 * factor * factor); + int h_bar = std::max(static_cast(factor), static_cast(std::round(height / factor) * factor)); + int w_bar = std::max(static_cast(factor), static_cast(std::round(width / factor) * factor)); + + if (std::max(height, width) > 200 * std::min(height, width)) { + LOG_WARN("LingBotVideo image aspect ratio is very large: %dx%d", width, height); + } + if (h_bar * w_bar > max_pixels) { + double beta = std::sqrt((height * width) / static_cast(max_pixels)); + h_bar = std::max(static_cast(factor), + static_cast(std::floor(height / beta / factor)) * static_cast(factor)); + w_bar = std::max(static_cast(factor), + static_cast(std::floor(width / beta / factor)) * static_cast(factor)); + } else if (h_bar * w_bar < min_pixels) { + double beta = std::sqrt(static_cast(min_pixels) / (height * width)); + h_bar = static_cast(std::ceil(height * beta / factor)) * static_cast(factor); + w_bar = static_cast(std::ceil(width * beta / factor)) * static_cast(factor); + } + + LOG_DEBUG("resize LingBotVideo ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar); + auto resized_image = clip_preprocess(image, w_bar, h_bar); + auto image_embed = llm->encode_image(n_threads, resized_image, false, true, true); + GGML_ASSERT(!image_embed.empty()); + + std::string image_prefix = prompt + img_prompt + "<|vision_start|>"; + int image_embed_idx = static_cast(tokenizer->encode(image_prefix, nullptr).size()); + image_embeds.emplace_back(image_embed_idx, image_embed); + + img_prompt += "<|vision_start|>"; + int64_t num_image_tokens = image_embed.shape()[1]; + img_prompt.reserve(img_prompt.size() + static_cast(num_image_tokens) * placeholder.size() + 32); + for (int j = 0; j < num_image_tokens; j++) { + img_prompt += placeholder; + } + img_prompt += "<|vision_end|>"; + } + prompt += img_prompt; + } + + prompt += conditioner_params.text; + prompt_attn_range = {0, 0}; + prompt += "<|im_end|>\n<|im_start|>assistant\n"; + } else if (sd_version_is_qwen_image(version)) { if (llm->enable_vision && conditioner_params.ref_images != nullptr && !conditioner_params.ref_images->empty()) { LOG_INFO("QwenImageEditPlusPipeline"); prompt_template_encode_start_idx = 64; diff --git a/src/model.h b/src/model.h index 2de82006a..cb2b2039e 100644 --- a/src/model.h +++ b/src/model.h @@ -35,6 +35,7 @@ enum SDVersion { VERSION_WAN2, VERSION_WAN2_2_I2V, VERSION_WAN2_2_TI2V, + VERSION_LINGBOT_VIDEO, VERSION_QWEN_IMAGE, VERSION_QWEN_IMAGE_LAYERED, VERSION_ANIMA, @@ -127,6 +128,13 @@ static inline bool sd_version_is_wan(SDVersion version) { return false; } +static inline bool sd_version_is_lingbot_video(SDVersion version) { + if (version == VERSION_LINGBOT_VIDEO) { + return true; + } + return false; +} + static inline bool sd_version_is_qwen_image(SDVersion version) { if (version == VERSION_QWEN_IMAGE || version == VERSION_QWEN_IMAGE_LAYERED) { return true; @@ -226,7 +234,7 @@ static inline bool sd_version_uses_flux2_vae(SDVersion version) { } static inline bool sd_version_uses_wan_vae(SDVersion version) { - if (sd_version_is_wan(version) || sd_version_is_qwen_image(version) || sd_version_is_krea2(version) || sd_version_is_anima(version)) { + if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_qwen_image(version) || sd_version_is_krea2(version) || sd_version_is_anima(version)) { return true; } return false; @@ -249,6 +257,7 @@ static inline bool sd_version_is_dit(SDVersion version) { sd_version_is_ltxav(version) || sd_version_is_sd3(version) || sd_version_is_wan(version) || + sd_version_is_lingbot_video(version) || sd_version_is_qwen_image(version) || version == VERSION_HIDREAM_O1 || sd_version_is_anima(version) || diff --git a/src/model/common/rope.hpp b/src/model/common/rope.hpp index d6cbb2e6a..04ecde3c5 100644 --- a/src/model/common/rope.hpp +++ b/src/model/common/rope.hpp @@ -759,6 +759,40 @@ namespace Rope { return embed_nd(ids, bs, static_cast(theta), axes_dim); } + __STATIC_INLINE__ std::vector> gen_lingbot_video_ids(int t, + int h, + int w, + int pt, + int ph, + int pw, + int bs, + int context_len) { + auto vid_ids_repeated = gen_vid_ids(t, h, w, pt, ph, pw, bs, context_len + 1); + + std::vector> txt_ids(bs * context_len, std::vector(3, 0.0f)); + for (int i = 0; i < bs; ++i) { + for (int j = 0; j < context_len; ++j) { + txt_ids[i * context_len + j][0] = static_cast(j + 1); + } + } + + return concat_ids(vid_ids_repeated, txt_ids, bs); + } + + __STATIC_INLINE__ std::vector gen_lingbot_video_pe(int t, + int h, + int w, + int pt, + int ph, + int pw, + int bs, + int context_len, + int theta, + const std::vector& axes_dim) { + std::vector> ids = gen_lingbot_video_ids(t, h, w, pt, ph, pw, bs, context_len); + return embed_nd(ids, bs, static_cast(theta), axes_dim); + } + __STATIC_INLINE__ std::vector> gen_qwen2vl_ids(int grid_h, int grid_w, int merge_size, diff --git a/src/model/diffusion/dit.hpp b/src/model/diffusion/dit.hpp index a88e7546d..e7d7b67f8 100644 --- a/src/model/diffusion/dit.hpp +++ b/src/model/diffusion/dit.hpp @@ -104,14 +104,15 @@ namespace DiT { return x; } - inline ggml_tensor* patchify(ggml_context* ctx, - ggml_tensor* x, - int pt, - int ph, - int pw, - int64_t N = 1) { + inline ggml_tensor* patchify_3d(ggml_context* ctx, + ggml_tensor* x, + int pt, + int ph, + int pw, + int64_t N = 1, + bool patch_last = true) { // x: [N*C, T, H, W] - // return: [N, h*w, C*pt*ph*pw] + // return: [N, t_len*h_len*w_len, C*pt*ph*pw] if patch_last else [N, t_len*h_len*w_len, C*pt*ph*pw] or [N, t_len*h_len*w_len, pt*ph*pw*C] int64_t C = x->ne[3] / N; int64_t T = x->ne[2]; int64_t H = x->ne[1]; @@ -123,15 +124,20 @@ namespace DiT { GGML_ASSERT(C * N == x->ne[3]); GGML_ASSERT(t_len * pt == T && h_len * ph == H && w_len * pw == W); - x = ggml_reshape_4d(ctx, x, pw * w_len, ph * h_len, pt, t_len * C * N); // [N*C*t_len, pt, h_len*ph, w_len*pw] - x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len, h_len*ph, pt, w_len*pw] - x = ggml_reshape_4d(ctx, x, pw * w_len, pt, ph, h_len * t_len * C * N); // [N*C*t_len*h_len, ph, pt, w_len*pw] - x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len*h_len, pt, ph, w_len*pw] - x = ggml_reshape_4d(ctx, x, pw, w_len, ph * pt, h_len * t_len * C * N); // [N*C*t_len*h_len, pt*ph, w_len, pw] - x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len*h_len, w_len, pt*ph, pw] - x = ggml_reshape_4d(ctx, x, pw * ph * pt, w_len * h_len * t_len, C, N); // [N, C, t_len*h_len*w_len, pt*ph*pw] - x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N, t_len*h_len*w_len, C, pt*ph*pw] - x = ggml_reshape_4d(ctx, x, pw * ph * pt * C, w_len * h_len * t_len, N, 1); // [N, t_len*h_len*w_len, C*pt*ph*pw] + x = ggml_reshape_4d(ctx, x, pw * w_len, ph * h_len, pt, t_len * C * N); // [N*C*t_len, pt, h_len*ph, w_len*pw] + x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len, h_len*ph, pt, w_len*pw] + x = ggml_reshape_4d(ctx, x, pw * w_len, pt, ph, h_len * t_len * C * N); // [N*C*t_len*h_len, ph, pt, w_len*pw] + x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len*h_len, pt, ph, w_len*pw] + x = ggml_reshape_4d(ctx, x, pw, w_len, ph * pt, h_len * t_len * C * N); // [N*C*t_len*h_len, pt*ph, w_len, pw] + x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len*h_len, w_len, pt*ph, pw] + x = ggml_reshape_4d(ctx, x, pw * ph * pt, w_len * h_len * t_len, C, N); // [N, C, t_len*h_len*w_len, pt*ph*pw] + if (patch_last) { + x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N, t_len*h_len*w_len, C, pt*ph*pw] + } else { + x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 2, 0, 1, 3)); // [N, t_len*h_len*w_len, pt*ph*pw, C] + } + // [N, t_len*h_len*w_len, C*pt*ph*pw] or [N, t_len*h_len*w_len, pt*ph*pw*C] + x = ggml_reshape_4d(ctx, x, pw * ph * pt * C, w_len * h_len * t_len, N, 1); return x; } @@ -142,16 +148,23 @@ namespace DiT { int64_t w_len, int pt, int ph, - int pw) { - // x: [N, t_len*h_len*w_len, C*pt*ph*pw] + int pw, + bool patch_last = true) { + // x: [N, t_len*h_len*w_len, C*pt*ph*pw] if patch_last else [N, t_len*h_len*w_len, pt*ph*pw*C] // return: [N*C, t_len*pt, h_len*ph, w_len*pw] int64_t N = x->ne[2]; int64_t C = x->ne[0] / pt / ph / pw; GGML_ASSERT(C * pt * ph * pw == x->ne[0]); - x = ggml_reshape_4d(ctx, x, pw * ph * pt, C, w_len * h_len * t_len, N); // [N, t_len*h_len*w_len, C, pt*ph*pw] - x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N, C, t_len*h_len*w_len, pt*ph*pw] + if (patch_last) { + x = ggml_reshape_4d(ctx, x, pw * ph * pt, C, w_len * h_len * t_len, N); // [N, t_len*h_len*w_len, C, pt*ph*pw] + x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N, C, t_len*h_len*w_len, pt*ph*pw] + } else { + x = ggml_reshape_4d(ctx, x, C, pw * ph * pt, w_len * h_len * t_len, N); // [N, t_len*h_len*w_len, pt*ph*pw, C] + x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 1, 2, 0, 3)); // [N, C, t_len*h_len*w_len, pt*ph*pw] + } + x = ggml_reshape_4d(ctx, x, pw, ph * pt, w_len, h_len * t_len * C * N); // [N*C*t_len*h_len, w_len, pt*ph, pw] x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len*h_len, pt*ph, w_len, pw] x = ggml_reshape_4d(ctx, x, pw * w_len, ph, pt, h_len * t_len * C * N); // [N*C*t_len*h_len, pt, ph, w_len*pw] diff --git a/src/model/diffusion/lingbot_video.hpp b/src/model/diffusion/lingbot_video.hpp new file mode 100644 index 000000000..1868daa05 --- /dev/null +++ b/src/model/diffusion/lingbot_video.hpp @@ -0,0 +1,692 @@ +#ifndef __SD_MODEL_DIFFUSION_LINGBOT_VIDEO_HPP__ +#define __SD_MODEL_DIFFUSION_LINGBOT_VIDEO_HPP__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/util.h" +#include "model/common/rope.hpp" +#include "model/diffusion/dit.hpp" +#include "model/diffusion/flux.hpp" +#include "model/diffusion/model.hpp" +#include "model/diffusion/qwen_image.hpp" + +namespace LingBotVideo { + constexpr int LINGBOT_VIDEO_GRAPH_SIZE = 65536; + + struct LingBotVideoConfig { + int patch_t = 1; + int patch_h = 2; + int patch_w = 2; + int64_t in_channels = 16; + int64_t out_channels = 16; + int64_t hidden_size = 2048; + int64_t num_attention_heads = 16; + int64_t depth = 24; + int64_t intermediate_size = 6144; + int64_t text_dim = 2560; + int64_t freq_dim = 256; + float norm_eps = 1e-6f; + int rope_theta = 256; + std::vector axes_dim = {32, 48, 48}; + int axes_dim_sum = 128; + bool qkv_bias = false; + bool out_bias = true; + bool patch_embed_bias = true; + bool timestep_mlp_bias = true; + int64_t num_experts = 0; + int64_t num_experts_per_tok = 8; + int64_t moe_intermediate_size = 512; + int64_t decoder_sparse_step = 1; + int64_t n_shared_experts = 0; + bool norm_topk_prob = true; + float routed_scaling_factor = 1.0f; + int64_t n_group = 0; + int64_t topk_group = 0; + std::set sparse_layers; + + static LingBotVideoConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, + const std::string& prefix) { + LingBotVideoConfig config; + config.depth = 0; + + for (const auto& [name, tensor_storage] : tensor_storage_map) { + if (!starts_with(name, prefix)) { + continue; + } + + if (ends_with(name, "patch_embedder.weight") && tensor_storage.n_dims == 2) { + int64_t patch_dim = tensor_storage.ne[0]; + config.hidden_size = tensor_storage.ne[1]; + int64_t patch_volume = config.patch_t * config.patch_h * config.patch_w; + if (patch_dim % patch_volume == 0) { + config.in_channels = patch_dim / patch_volume; + } + } else if (ends_with(name, "text_embedder.linear_1.weight") && tensor_storage.n_dims == 2) { + config.text_dim = tensor_storage.ne[0]; + } else if (ends_with(name, "time_embedder.linear_1.weight") && tensor_storage.n_dims == 2) { + config.freq_dim = tensor_storage.ne[0]; + } else if (ends_with(name, "blocks.0.attn.norm_q.weight") && tensor_storage.n_dims == 1) { + int64_t head_dim = tensor_storage.ne[0]; + if (head_dim > 0) { + config.num_attention_heads = config.hidden_size / head_dim; + } + } else if (name.find(".attn.to_q.bias") != std::string::npos) { + config.qkv_bias = true; + } else if (name.find(".ffn.gate_proj.weight") != std::string::npos && tensor_storage.n_dims == 2) { + config.intermediate_size = tensor_storage.ne[1]; + } else if (name.find(".ffn.experts.w1") != std::string::npos && tensor_storage.n_dims == 3) { + config.num_experts = tensor_storage.ne[2]; + config.moe_intermediate_size = tensor_storage.ne[1]; + } else if (name.find(".ffn.shared_experts.gate_proj.weight") != std::string::npos && tensor_storage.n_dims == 2) { + if (config.moe_intermediate_size > 0) { + config.n_shared_experts = tensor_storage.ne[1] / config.moe_intermediate_size; + } + } else if (ends_with(name, "proj_out.weight") && tensor_storage.n_dims == 2) { + int64_t out_dim = tensor_storage.ne[1]; + int64_t patch_volume = config.patch_t * config.patch_h * config.patch_w; + config.out_channels = patch_volume > 0 ? out_dim / patch_volume : config.out_channels; + } + + size_t block_pos = name.find("blocks."); + if (block_pos != std::string::npos) { + auto items = split_string(name.substr(block_pos), '.'); + if (items.size() > 1) { + int block_index = atoi(items[1].c_str()); + if (block_index + 1 > config.depth) { + config.depth = block_index + 1; + } + if (name.find("blocks." + std::to_string(block_index) + ".ffn.experts.w1") != std::string::npos) { + config.sparse_layers.insert(block_index); + } + } + } + } + + if (config.depth == 0) { + config.depth = 24; + } + config.axes_dim_sum = 0; + for (int axis_dim : config.axes_dim) { + config.axes_dim_sum += axis_dim; + } + if (!config.sparse_layers.empty()) { + config.num_experts = 128; + config.num_experts_per_tok = 8; + config.moe_intermediate_size = 768; + config.decoder_sparse_step = 1; + config.n_shared_experts = 1; + config.norm_topk_prob = true; + config.n_group = 4; + config.topk_group = 2; + config.routed_scaling_factor = 2.5f; + } + LOG_DEBUG("lingbot_video: depth = %" PRId64 ", hidden_size = %" PRId64 ", heads = %" PRId64 ", text_dim = %" PRId64 ", experts = %" PRId64 ", experts_per_tok = %" PRId64 ", n_group = %" PRId64 ", topk_group = %" PRId64 ", route_scale = %.2f, sparse_layers = %zu", + config.depth, + config.hidden_size, + config.num_attention_heads, + config.text_dim, + config.num_experts, + config.num_experts_per_tok, + config.n_group, + config.topk_group, + config.routed_scaling_factor, + config.sparse_layers.size()); + return config; + } + }; + + struct LingBotVideoTextEmbedder : public GGMLBlock { + LingBotVideoTextEmbedder(int64_t text_dim, + int64_t hidden_size, + float eps = 1e-6f) { + blocks["norm"] = std::make_shared(text_dim, eps); + blocks["linear_1"] = std::make_shared(text_dim, hidden_size, true); + blocks["linear_2"] = std::make_shared(hidden_size, hidden_size, true); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + auto norm = std::dynamic_pointer_cast(blocks["norm"]); + auto linear_1 = std::dynamic_pointer_cast(blocks["linear_1"]); + auto linear_2 = std::dynamic_pointer_cast(blocks["linear_2"]); + + x = norm->forward(ctx, x); + x = linear_1->forward(ctx, x); + x = ggml_silu_inplace(ctx->ggml_ctx, x); + x = linear_2->forward(ctx, x); + return x; + } + }; + + struct LingBotVideoAttention : public GGMLBlock { + int64_t num_heads; + int64_t head_dim; + + LingBotVideoAttention(int64_t hidden_size, + int64_t num_heads, + bool qkv_bias = false, + bool out_bias = true, + float eps = 1e-6f) + : num_heads(num_heads), + head_dim(hidden_size / num_heads) { + int64_t inner_dim = num_heads * head_dim; + blocks["to_q"] = std::make_shared(hidden_size, inner_dim, qkv_bias); + blocks["to_k"] = std::make_shared(hidden_size, inner_dim, qkv_bias); + blocks["to_v"] = std::make_shared(hidden_size, inner_dim, qkv_bias); + blocks["norm_q"] = std::make_shared(head_dim, eps); + blocks["norm_k"] = std::make_shared(head_dim, eps); + blocks["to_out"] = std::make_shared(inner_dim, hidden_size, out_bias); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, + ggml_tensor* x, + ggml_tensor* pe, + ggml_tensor* attention_mask = nullptr) { + // x: [N, video_tokens + text_tokens, hidden_size] + auto to_q = std::dynamic_pointer_cast(blocks["to_q"]); + auto to_k = std::dynamic_pointer_cast(blocks["to_k"]); + auto to_v = std::dynamic_pointer_cast(blocks["to_v"]); + auto norm_q = std::dynamic_pointer_cast(blocks["norm_q"]); + auto norm_k = std::dynamic_pointer_cast(blocks["norm_k"]); + auto to_out = std::dynamic_pointer_cast(blocks["to_out"]); + + int64_t S = x->ne[1]; + int64_t N = x->ne[2]; + + auto q = to_q->forward(ctx, x); + auto k = to_k->forward(ctx, x); + auto v = to_v->forward(ctx, x); + + q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim, num_heads, S, N); + k = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim, num_heads, S, N); + v = ggml_reshape_4d(ctx->ggml_ctx, v, head_dim, num_heads, S, N); + + q = norm_q->forward(ctx, q); + k = norm_k->forward(ctx, k); + + x = Rope::attention(ctx, q, k, v, pe, attention_mask); + x = to_out->forward(ctx, x); + return x; + } + }; + + struct LingBotVideoMLP : public UnaryBlock { + LingBotVideoMLP(int64_t hidden_size, + int64_t intermediate_size) { + blocks["gate_proj"] = std::make_shared(hidden_size, intermediate_size, false); + blocks["up_proj"] = std::make_shared(hidden_size, intermediate_size, false); + blocks["down_proj"] = std::make_shared(intermediate_size, hidden_size, false); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + auto gate_proj = std::dynamic_pointer_cast(blocks["gate_proj"]); + auto up_proj = std::dynamic_pointer_cast(blocks["up_proj"]); + auto down_proj = std::dynamic_pointer_cast(blocks["down_proj"]); + + auto gate = gate_proj->forward(ctx, x); + gate = ggml_silu_inplace(ctx->ggml_ctx, gate); + auto up = up_proj->forward(ctx, x); + x = ggml_mul(ctx->ggml_ctx, gate, up); + x = down_proj->forward(ctx, x); + return x; + } + }; + + struct LingBotVideoSparseMoeBlock : public UnaryBlock { + int64_t hidden_size; + int64_t intermediate_size; + int64_t num_experts; + int64_t num_experts_per_tok; + bool has_shared_experts; + bool norm_topk_prob; + float routed_scaling_factor; + int64_t n_group; + int64_t topk_group; + std::vector group_expert_mask_vec; + bool has_correction_bias = false; + + LingBotVideoSparseMoeBlock(const LingBotVideoConfig& config) + : hidden_size(config.hidden_size), + intermediate_size(config.moe_intermediate_size), + num_experts(config.num_experts), + num_experts_per_tok(config.num_experts_per_tok), + has_shared_experts(config.n_shared_experts > 0), + norm_topk_prob(config.norm_topk_prob), + routed_scaling_factor(config.routed_scaling_factor), + n_group(config.n_group), + topk_group(config.topk_group) { + if (n_group > 1) { + GGML_ASSERT(num_experts % n_group == 0); + int64_t experts_per_group = num_experts / n_group; + group_expert_mask_vec.assign(static_cast(num_experts * n_group), 0.f); + for (int64_t group = 0; group < n_group; ++group) { + int64_t expert_begin = group * experts_per_group; + int64_t expert_end = expert_begin + experts_per_group; + for (int64_t expert = expert_begin; expert < expert_end; ++expert) { + group_expert_mask_vec[static_cast(group * num_experts + expert)] = 1.f; + } + } + } + if (has_shared_experts) { + blocks["shared_experts"] = std::make_shared(hidden_size, + intermediate_size * config.n_shared_experts); + } + } + + void init_params(ggml_context* ctx, + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "") override { + ggml_type router_type = get_type(prefix + "router.weight", tensor_storage_map, GGML_TYPE_F32); + ggml_type w1_type = get_type(prefix + "experts.w1", tensor_storage_map, GGML_TYPE_F32); + ggml_type w2_type = get_type(prefix + "experts.w2", tensor_storage_map, GGML_TYPE_F32); + ggml_type w3_type = get_type(prefix + "experts.w3", tensor_storage_map, GGML_TYPE_F32); + params["router.weight"] = ggml_new_tensor_2d(ctx, router_type, hidden_size, num_experts); + if (tensor_storage_map.find(prefix + "router.e_score_correction_bias") != tensor_storage_map.end()) { + params["router.e_score_correction_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, num_experts); + has_correction_bias = true; + } + params["experts.w1"] = ggml_new_tensor_3d(ctx, w1_type, hidden_size, intermediate_size, num_experts); + params["experts.w2"] = ggml_new_tensor_3d(ctx, w2_type, intermediate_size, hidden_size, num_experts); + params["experts.w3"] = ggml_new_tensor_3d(ctx, w3_type, hidden_size, intermediate_size, num_experts); + } + + ggml_tensor* expert_linear(GGMLRunnerContext* ctx, + const std::string& weight_name, + ggml_tensor* x, + ggml_tensor* selected_experts) { + return ggml_mul_mat_id(ctx->ggml_ctx, params[weight_name], x, selected_experts); + } + + ggml_tensor* select_experts(GGMLRunnerContext* ctx, ggml_tensor* choice_scores) { + ggml_context* gctx = ctx->ggml_ctx; + if (n_group <= 1 || topk_group <= 0) { + return ggml_argsort_top_k(gctx, choice_scores, static_cast(num_experts_per_tok)); + } + + GGML_ASSERT(choice_scores->ne[0] == num_experts); + GGML_ASSERT(num_experts % n_group == 0); + GGML_ASSERT(topk_group > 0 && topk_group <= n_group); + GGML_ASSERT(!group_expert_mask_vec.empty()); + + const int64_t n_token_total = choice_scores->ne[1]; + const int64_t experts_per_group = num_experts / n_group; + const int group_score_k = 2; + GGML_ASSERT(experts_per_group >= group_score_k); + + ggml_tensor* grouped_scores = ggml_reshape_3d(gctx, choice_scores, experts_per_group, n_group, n_token_total); + ggml_tensor* group_top_ids = ggml_argsort_top_k(gctx, grouped_scores, group_score_k); + grouped_scores = ggml_reshape_3d(gctx, grouped_scores, 1, experts_per_group, n_group * n_token_total); + group_top_ids = ggml_cont(gctx, group_top_ids); + group_top_ids = ggml_reshape_2d(gctx, group_top_ids, group_score_k, n_group * n_token_total); + + ggml_tensor* group_top_values = ggml_get_rows(gctx, grouped_scores, group_top_ids); + group_top_values = ggml_reshape_3d(gctx, group_top_values, group_score_k, n_group, n_token_total); + + ggml_tensor* group_scores = nullptr; + for (int rank = 0; rank < group_score_k; ++rank) { + ggml_tensor* value = ggml_view_3d(gctx, + group_top_values, + 1, + n_group, + n_token_total, + group_top_values->nb[1], + group_top_values->nb[2], + rank * group_top_values->nb[0]); + group_scores = group_scores == nullptr ? value : ggml_add(gctx, group_scores, value); + } + group_scores = ggml_reshape_2d(gctx, group_scores, n_group, n_token_total); + + ggml_tensor* selected_groups = ggml_argsort_top_k(gctx, group_scores, static_cast(topk_group)); + selected_groups = ggml_cont(gctx, selected_groups); + + ggml_tensor* group_expert_mask = ggml_new_tensor_3d(gctx, GGML_TYPE_F32, num_experts, n_group, 1); + ctx->bind_backend_tensor_data(group_expert_mask, group_expert_mask_vec.data()); + ggml_tensor* group_expert_mask_template = ggml_new_tensor_3d(gctx, GGML_TYPE_F32, num_experts, n_group, n_token_total); + group_expert_mask = ggml_repeat(gctx, group_expert_mask, group_expert_mask_template); + + ggml_tensor* selected_group_masks = ggml_get_rows(gctx, group_expert_mask, selected_groups); + ggml_tensor* selected_mask = nullptr; + for (int64_t rank = 0; rank < topk_group; ++rank) { + ggml_tensor* mask = ggml_view_3d(gctx, + selected_group_masks, + num_experts, + 1, + n_token_total, + selected_group_masks->nb[1], + selected_group_masks->nb[2], + rank * selected_group_masks->nb[1]); + selected_mask = selected_mask == nullptr ? mask : ggml_add(gctx, selected_mask, mask); + } + selected_mask = ggml_reshape_2d(gctx, selected_mask, num_experts, n_token_total); + + ggml_tensor* excluded_group_mask = ggml_sub(gctx, selected_mask, ggml_ext_ones_like(gctx, selected_mask)); + ggml_tensor* masked_scores = ggml_add(gctx, choice_scores, ggml_scale(gctx, excluded_group_mask, 1.0e9f)); + return ggml_argsort_top_k(gctx, masked_scores, static_cast(num_experts_per_tok)); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + // x: [N, tokens, hidden_size] + GGML_ASSERT(num_experts > 0); + GGML_ASSERT(num_experts_per_tok > 0 && num_experts_per_tok <= num_experts); + + ggml_tensor* residual = x; + const int64_t n_token = x->ne[1]; + const int64_t N = x->ne[2]; + const int64_t n_token_total = n_token * N; + + ggml_tensor* router_logits = ggml_mul_mat(ctx->ggml_ctx, params["router.weight"], x); + router_logits = ggml_reshape_2d(ctx->ggml_ctx, router_logits, num_experts, n_token_total); + ggml_tensor* probs = ggml_sigmoid(ctx->ggml_ctx, router_logits); + ggml_tensor* choice_scores = probs; + if (has_correction_bias) { + choice_scores = ggml_add(ctx->ggml_ctx, choice_scores, params["router.e_score_correction_bias"]); + } + + ggml_tensor* selected_experts = select_experts(ctx, choice_scores); + ggml_tensor* weights = ggml_get_rows(ctx->ggml_ctx, + ggml_reshape_3d(ctx->ggml_ctx, probs, 1, num_experts, n_token_total), + selected_experts); + weights = ggml_reshape_2d(ctx->ggml_ctx, weights, num_experts_per_tok, n_token_total); + if (norm_topk_prob && num_experts_per_tok > 1) { + auto weights_sum = ggml_sum_rows(ctx->ggml_ctx, weights); + weights_sum = ggml_clamp(ctx->ggml_ctx, weights_sum, 6.103515625e-5f, INFINITY); + weights = ggml_div(ctx->ggml_ctx, weights, weights_sum); + } + if (routed_scaling_factor != 1.0f) { + weights = ggml_scale(ctx->ggml_ctx, weights, routed_scaling_factor); + } + weights = ggml_reshape_3d(ctx->ggml_ctx, weights, 1, num_experts_per_tok, n_token_total); + + x = ggml_reshape_3d(ctx->ggml_ctx, x, hidden_size, 1, n_token_total); + auto gate = expert_linear(ctx, "experts.w1", x, selected_experts); + gate = ggml_silu_inplace(ctx->ggml_ctx, gate); + auto up = expert_linear(ctx, "experts.w3", x, selected_experts); + auto act = ggml_mul(ctx->ggml_ctx, gate, up); + auto out = expert_linear(ctx, "experts.w2", act, selected_experts); + out = ggml_mul(ctx->ggml_ctx, out, weights); + + ggml_tensor* summed = nullptr; + for (int64_t i = 0; i < num_experts_per_tok; ++i) { + auto expert_out = ggml_view_2d(ctx->ggml_ctx, + out, + hidden_size, + n_token_total, + out->nb[2], + i * out->nb[1]); + summed = summed == nullptr ? expert_out : ggml_add(ctx->ggml_ctx, summed, expert_out); + } + if (num_experts_per_tok == 1) { + summed = ggml_cont(ctx->ggml_ctx, summed); + } + summed = ggml_reshape_3d(ctx->ggml_ctx, summed, hidden_size, n_token, N); + + if (has_shared_experts) { + auto shared_experts = std::dynamic_pointer_cast(blocks["shared_experts"]); + summed = ggml_add(ctx->ggml_ctx, summed, shared_experts->forward(ctx, residual)); + } + return summed; + } + }; + + struct LingBotVideoBlock : public GGMLBlock { + int64_t hidden_size; + + LingBotVideoBlock(const LingBotVideoConfig& config, + bool sparse) + : hidden_size(config.hidden_size) { + blocks["norm1"] = std::make_shared(config.hidden_size, config.norm_eps); + blocks["attn"] = std::make_shared(config.hidden_size, + config.num_attention_heads, + config.qkv_bias, + config.out_bias, + config.norm_eps); + blocks["norm_post_attn"] = std::make_shared(config.hidden_size, config.norm_eps); + blocks["norm2"] = std::make_shared(config.hidden_size, config.norm_eps); + if (sparse) { + blocks["ffn"] = std::make_shared(config); + } else { + blocks["ffn"] = std::make_shared(config.hidden_size, config.intermediate_size); + } + blocks["norm_post_ffn"] = std::make_shared(config.hidden_size, config.norm_eps); + } + + void init_params(ggml_context* ctx, + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "") override { + params["scale_shift_table"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size * 6, 1); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, + ggml_tensor* x, + ggml_tensor* temb6, + ggml_tensor* pe, + ggml_tensor* attention_mask = nullptr) { + // x: [N, tokens, hidden_size], temb6: [N, tokens, 6 * hidden_size] + auto norm1 = std::dynamic_pointer_cast(blocks["norm1"]); + auto attn = std::dynamic_pointer_cast(blocks["attn"]); + auto norm_post_attn = std::dynamic_pointer_cast(blocks["norm_post_attn"]); + auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); + auto ffn = std::dynamic_pointer_cast(blocks["ffn"]); + auto norm_post_ffn = std::dynamic_pointer_cast(blocks["norm_post_ffn"]); + + auto table = ggml_reshape_3d(ctx->ggml_ctx, params["scale_shift_table"], hidden_size * 6, 1, 1); + auto mod = ggml_add(ctx->ggml_ctx, temb6, table); + auto mods = ggml_ext_chunk(ctx->ggml_ctx, mod, 6, 0); + + auto shift_msa = mods[0]; + auto scale_msa = mods[1]; + auto gate_msa = ggml_tanh(ctx->ggml_ctx, mods[2]); + auto shift_mlp = mods[3]; + auto scale_mlp = mods[4]; + auto gate_mlp = ggml_tanh(ctx->ggml_ctx, mods[5]); + + auto attn_in = Flux::modulate(ctx->ggml_ctx, norm1->forward(ctx, x), shift_msa, scale_msa, true); + auto attn_out = attn->forward(ctx, attn_in, pe, attention_mask); + attn_out = norm_post_attn->forward(ctx, attn_out); + x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, gate_msa, attn_out)); + + auto ffn_in = Flux::modulate(ctx->ggml_ctx, norm2->forward(ctx, x), shift_mlp, scale_mlp, true); + auto ffn_out = ffn->forward(ctx, ffn_in); + ffn_out = norm_post_ffn->forward(ctx, ffn_out); + x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, gate_mlp, ffn_out)); + return x; + } + }; + + class LingBotVideoModel : public GGMLBlock { + public: + LingBotVideoConfig config; + + LingBotVideoModel() = default; + LingBotVideoModel(LingBotVideoConfig config) + : config(config) { + int64_t patch_dim = config.patch_t * config.patch_h * config.patch_w * config.in_channels; + blocks["patch_embedder"] = std::make_shared(patch_dim, config.hidden_size, config.patch_embed_bias); + blocks["time_embedder"] = std::make_shared(config.freq_dim, + config.hidden_size, + config.hidden_size, + 0, + config.timestep_mlp_bias); + blocks["time_modulation.1"] = std::make_shared(config.hidden_size, 6 * config.hidden_size, true); + blocks["text_embedder"] = std::make_shared(config.text_dim, + config.hidden_size, + config.norm_eps); + for (int i = 0; i < config.depth; i++) { + bool sparse = config.sparse_layers.find(i) != config.sparse_layers.end(); + blocks["blocks." + std::to_string(i)] = std::make_shared(config, sparse); + } + blocks["norm_out"] = std::make_shared(config.hidden_size, config.norm_eps, false); + blocks["norm_out_modulation.1"] = std::make_shared(config.hidden_size, 2 * config.hidden_size, true); + blocks["proj_out"] = std::make_shared(config.hidden_size, + config.patch_t * config.patch_h * config.patch_w * config.out_channels, + true); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, + ggml_tensor* x, + ggml_tensor* timestep, + ggml_tensor* context, + ggml_tensor* pe) { + // x: [N*C, T, H, W], context: [N, text_tokens, text_dim] + GGML_ASSERT(context != nullptr); + GGML_ASSERT(x->ne[3] == config.in_channels); + GGML_ASSERT(x->ne[2] % config.patch_t == 0); + GGML_ASSERT(x->ne[1] % config.patch_h == 0); + GGML_ASSERT(x->ne[0] % config.patch_w == 0); + + auto patch_embedder = std::dynamic_pointer_cast(blocks["patch_embedder"]); + auto time_embedder = std::dynamic_pointer_cast(blocks["time_embedder"]); + auto time_modulation = std::dynamic_pointer_cast(blocks["time_modulation.1"]); + auto text_embedder = std::dynamic_pointer_cast(blocks["text_embedder"]); + auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); + auto norm_out_modulation = std::dynamic_pointer_cast(blocks["norm_out_modulation.1"]); + auto proj_out = std::dynamic_pointer_cast(blocks["proj_out"]); + + int64_t W = x->ne[0]; + int64_t H = x->ne[1]; + int64_t T = x->ne[2]; + int64_t N = 1; + int64_t t_len = T / config.patch_t; + int64_t h_len = H / config.patch_h; + int64_t w_len = W / config.patch_w; + int64_t n_img = t_len * h_len * w_len; + + auto img = DiT::patchify_3d(ctx->ggml_ctx, x, config.patch_t, config.patch_h, config.patch_w, N, false); + img = patch_embedder->forward(ctx, img); + + auto txt = text_embedder->forward(ctx, context); + auto hidden_states = ggml_concat(ctx->ggml_ctx, img, txt, 1); + + auto timestep_proj = ggml_ext_timestep_embedding(ctx->ggml_ctx, + timestep, + static_cast(config.freq_dim), + 10000, + 1.0f); + auto t_emb = time_embedder->forward(ctx, timestep_proj); + auto temb_template = ggml_new_tensor_3d(ctx->ggml_ctx, t_emb->type, t_emb->ne[0], hidden_states->ne[1], t_emb->ne[1]); + auto temb_tokens = ggml_repeat(ctx->ggml_ctx, + ggml_reshape_3d(ctx->ggml_ctx, t_emb, t_emb->ne[0], 1, t_emb->ne[1]), + temb_template); + auto temb6 = time_modulation->forward(ctx, ggml_silu(ctx->ggml_ctx, temb_tokens)); + + sd::ggml_graph_cut::mark_graph_cut(hidden_states, "lingbot_video.prelude", "hidden_states"); + + for (int i = 0; i < config.depth; i++) { + auto block = std::dynamic_pointer_cast(blocks["blocks." + std::to_string(i)]); + hidden_states = block->forward(ctx, hidden_states, temb6, pe); + sd::ggml_graph_cut::mark_graph_cut(hidden_states, "lingbot_video.blocks." + std::to_string(i), "hidden_states"); + } + + auto final_mods = ggml_ext_chunk(ctx->ggml_ctx, + norm_out_modulation->forward(ctx, ggml_silu(ctx->ggml_ctx, temb_tokens)), + 2, + 0); + hidden_states = norm_out->forward(ctx, hidden_states); + hidden_states = Flux::modulate(ctx->ggml_ctx, hidden_states, final_mods[0], final_mods[1], true); + hidden_states = proj_out->forward(ctx, hidden_states); + hidden_states = ggml_ext_slice(ctx->ggml_ctx, hidden_states, 1, 0, n_img); + + auto out = DiT::unpatchify_3d(ctx->ggml_ctx, + hidden_states, + t_len, + h_len, + w_len, + config.patch_t, + config.patch_h, + config.patch_w, + false); + return out; + } + }; + + struct LingBotVideoRunner : public DiffusionModelRunner { + LingBotVideoConfig config; + LingBotVideoModel lingbot_video; + std::vector pe_vec; + + LingBotVideoRunner(ggml_backend_t backend, + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + std::shared_ptr weight_manager = nullptr, + const char* model_args = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), + config(LingBotVideoConfig::detect_from_weights(tensor_storage_map, prefix)) { + SD_UNUSED(model_args); + + lingbot_video = LingBotVideoModel(config); + lingbot_video.init(params_ctx, tensor_storage_map, prefix); + } + + std::string get_desc() override { + return "lingbot_video"; + } + + void get_param_tensors(std::map& tensors, const std::string& prefix) override { + lingbot_video.get_param_tensors(tensors, prefix); + } + + ggml_cgraph* build_graph(const sd::Tensor& x_tensor, + const sd::Tensor& timesteps_tensor, + const sd::Tensor& context_tensor) { + ggml_cgraph* gf = new_graph_custom(LINGBOT_VIDEO_GRAPH_SIZE); + ggml_tensor* x = make_input(x_tensor); + ggml_tensor* timesteps = make_input(timesteps_tensor); + GGML_ASSERT(x_tensor.dim() == 5); + GGML_ASSERT(x->ne[3] == config.in_channels); + GGML_ASSERT(!context_tensor.empty()); + ggml_tensor* context = make_input(context_tensor); + + int64_t W = x->ne[0]; + int64_t H = x->ne[1]; + int64_t T = x->ne[2]; + int64_t N = x_tensor.shape()[4]; + GGML_ASSERT(N == 1); + pe_vec = Rope::gen_lingbot_video_pe(static_cast(T), + static_cast(H), + static_cast(W), + config.patch_t, + config.patch_h, + config.patch_w, + static_cast(N), + static_cast(context->ne[1]), + config.rope_theta, + config.axes_dim); + int pos_len = static_cast(pe_vec.size() / config.axes_dim_sum / 2); + auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len); + set_backend_tensor_data(pe, pe_vec.data()); + + auto runner_ctx = get_context(); + ggml_tensor* out = lingbot_video.forward(&runner_ctx, x, timesteps, context, pe); + ggml_build_forward_expand(gf, out); + return gf; + } + + sd::Tensor compute(int n_threads, + const sd::Tensor& x, + const sd::Tensor& timesteps, + const sd::Tensor& context) { + auto get_graph = [&]() -> ggml_cgraph* { + return build_graph(x, timesteps, context); + }; + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); + } + + sd::Tensor compute(int n_threads, + const DiffusionParams& diffusion_params) override { + GGML_ASSERT(diffusion_params.x != nullptr); + GGML_ASSERT(diffusion_params.timesteps != nullptr); + return compute(n_threads, + *diffusion_params.x, + *diffusion_params.timesteps, + tensor_or_empty(diffusion_params.context)); + } + }; +} // namespace LingBotVideo + +#endif // __SD_MODEL_DIFFUSION_LINGBOT_VIDEO_HPP__ diff --git a/src/model/diffusion/qwen_image.hpp b/src/model/diffusion/qwen_image.hpp index 52edc0d56..01641377a 100644 --- a/src/model/diffusion/qwen_image.hpp +++ b/src/model/diffusion/qwen_image.hpp @@ -517,7 +517,7 @@ namespace Qwen { if (input->ne[3] == 1) { input = ggml_reshape_4d(ctx->ggml_ctx, input, input->ne[0], input->ne[1], 1, input->ne[2]); } - return DiT::patchify(ctx->ggml_ctx, input, 1, config.patch_size, config.patch_size, N); + return DiT::patchify_3d(ctx->ggml_ctx, input, 1, config.patch_size, config.patch_size, N); }; auto img = patchify_input(x); diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 51091d98f..d7a46deb5 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -511,6 +511,9 @@ SDVersion ModelLoader::get_sd_version() { if (tensor_storage.name.find("model.diffusion_model.blocks.0.cross_attn.norm_k.weight") != std::string::npos) { is_wan = true; } + if (tensor_storage.name.find("model.diffusion_model.patch_embedder.weight") != std::string::npos) { + return VERSION_LINGBOT_VIDEO; + } if (tensor_storage.name.find("model.diffusion_model.patch_embedding.weight") != std::string::npos) { patch_embedding_channels = tensor_storage.ne[3]; } diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 28628be7c..462b24738 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -32,6 +32,7 @@ #include "model/diffusion/ideogram4.hpp" #include "model/diffusion/krea2.hpp" #include "model/diffusion/lens.hpp" +#include "model/diffusion/lingbot_video.hpp" #include "model/diffusion/ltxv.hpp" #include "model/diffusion/minit2i.hpp" #include "model/diffusion/mmdit.hpp" @@ -87,6 +88,7 @@ const char* model_version_to_str[] = { "Wan 2.x", "Wan 2.2 I2V", "Wan 2.2 TI2V", + "LingBot Video", "Qwen Image", "Qwen Image Layered", "Anima", @@ -1004,6 +1006,25 @@ class StableDiffusionGGML { return false; } } + } else if (sd_version_is_lingbot_video(version)) { + bool enable_vision = false; + for (const auto& [name, _] : tensor_storage_map) { + if (starts_with(name, "text_encoders.llm.visual.")) { + enable_vision = true; + break; + } + } + cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), + tensor_storage_map, + version, + "", + enable_vision, + model_manager); + diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), + tensor_storage_map, + "model.diffusion_model", + model_manager, + sd_ctx_params->model_args); } else if (sd_version_is_qwen_image(version)) { bool enable_vision = version != VERSION_QWEN_IMAGE_LAYERED; cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), @@ -1503,6 +1524,7 @@ class StableDiffusionGGML { } } else if (sd_version_is_sd3(version) || sd_version_is_wan(version) || + sd_version_is_lingbot_video(version) || sd_version_is_qwen_image(version) || version == VERSION_HIDREAM_O1 || sd_version_is_anima(version) || @@ -2332,7 +2354,7 @@ class StableDiffusionGGML { sd::Tensor timesteps_tensor({static_cast(timesteps_vec.size())}, timesteps_vec); sd::Tensor guidance_tensor({1}, std::vector{guidance.distilled_guidance}); sd::Tensor noised_input = x * c_in; - if (!denoise_mask.empty() && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version))) { + if (!denoise_mask.empty() && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) { noised_input = noised_input * denoise_mask + init_latent * (1.0f - denoise_mask); } @@ -2578,7 +2600,7 @@ class StableDiffusionGGML { int get_diffusion_model_down_factor() { int down_factor = 8; // unet if (sd_version_is_dit(version)) { - if (sd_version_is_wan(version)) { + if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version)) { down_factor = 2; } else { down_factor = 1; @@ -2641,7 +2663,7 @@ class StableDiffusionGGML { int latent_frames = frames; if (sd_version_is_ltxav(version)) { latent_frames = ((frames - 1) / 8) + 1; - } else if (sd_version_is_wan(version)) { + } else if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version)) { latent_frames = ((frames - 1) / 4) + 1; } return latent_frames; @@ -2654,7 +2676,7 @@ class StableDiffusionGGML { if (sd_version_is_ltxav(version)) { return (latent_frames - 1) * 8 + 1; } - if (sd_version_is_wan(version)) { + if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version)) { return (latent_frames - 1) * 4 + 1; } return latent_frames; @@ -3328,7 +3350,7 @@ struct sd_ctx_t { }; static bool sd_version_supports_video_generation(SDVersion version) { - return version == VERSION_SVD || sd_version_is_wan(version) || sd_version_is_ltxav(version); + return version == VERSION_SVD || sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_ltxav(version); } static bool sd_version_supports_image_generation(SDVersion version) { @@ -5419,6 +5441,27 @@ static std::optional prepare_video_generation_latents(sd sd::ops::fill_slice(&latents.denoise_mask, 2, latents.init_latent.shape()[2] - 1, latents.init_latent.shape()[2], 0.0f); } + int64_t t2 = ggml_time_ms(); + LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1); + } else if (sd_version_is_lingbot_video(sd_ctx->sd->version) && !start_image.empty()) { + LOG_INFO("LingBot Video IMG2VID"); + + int64_t t1 = ggml_time_ms(); + auto init_img = start_image.reshape({start_image.shape()[0], start_image.shape()[1], 1, start_image.shape()[2], 1}); + auto init_image_latent = sd_ctx->sd->encode_first_stage(init_img); + if (init_image_latent.empty()) { + LOG_ERROR("failed to encode init video frame"); + return std::nullopt; + } + + latents.init_latent = sd_ctx->sd->generate_init_latent(request->width, request->height, request->frames, true); + sd::ops::slice_assign(&latents.init_latent, 2, 0, init_image_latent.shape()[2], init_image_latent); + + latents.denoise_mask = sd::full({latents.init_latent.shape()[0], latents.init_latent.shape()[1], latents.init_latent.shape()[2], 1, 1}, 1.f); + sd::ops::fill_slice(&latents.denoise_mask, 2, 0, init_image_latent.shape()[2], 0.0f); + + latents.ref_images.push_back(start_image); + int64_t t2 = ggml_time_ms(); LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1); } else if (sd_ctx->sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" || @@ -5524,6 +5567,7 @@ static ImageGenerationEmbeds prepare_video_generation_embeds(sd_ctx_t* sd_ctx, condition_params.clip_skip = request.clip_skip; condition_params.text = request.prompt; condition_params.zero_out_masked = true; + condition_params.ref_images = &latents.ref_images; int64_t prepare_start_ms = ggml_time_ms(); embeds.cond = sd_ctx->sd->cond_stage_model->get_learned_condition(sd_ctx->sd->n_threads,