diff --git a/src/api/consts/exceptions.ts b/src/api/consts/exceptions.ts index 5908b67..84b42cb 100644 --- a/src/api/consts/exceptions.ts +++ b/src/api/consts/exceptions.ts @@ -11,4 +11,5 @@ export default { API_VIDEO_GENERATION_FAILED: [-2008, '视频生成失败'], API_IMAGE_GENERATION_INSUFFICIENT_POINTS: [-2009, '即梦积分不足'], API_IMAGE_URL: [-2010, '即梦积分不足'], + API_HISTORY_EMPTY: [-2011, '生成结果为空'], } \ No newline at end of file diff --git a/src/api/controllers/video.ts b/src/api/controllers/video.ts index 5ce7b03..2fa8ac6 100644 --- a/src/api/controllers/video.ts +++ b/src/api/controllers/video.ts @@ -18,7 +18,7 @@ const MODEL_MAP = { export function getModel(model: string) { return MODEL_MAP[model] || MODEL_MAP[DEFAULT_MODEL]; } - +//图片生成视频 首帧 export async function generateVideo( _model: string, task_id: string, @@ -168,10 +168,11 @@ export async function generateVideo( ); const historyId = aigc_data.history_record_id; if (!historyId) - throw new APIException(EX.API_IMAGE_GENERATION_FAILED, "记录ID不存在"); + throw new APIException(EX.API_VIDEO_GENERATION_FAILED, "记录ID不存在"); let status = 20, failCode, item_list = []; //https://jimeng.jianying.com/mweb/v1/get_history_by_ids? // + let emptyCount = 30; while (status === 20) { await new Promise((resolve) => setTimeout(resolve, 1000)); const result = await request("post", "/mweb/v1/get_history_by_ids", refreshToken, { @@ -182,8 +183,16 @@ export async function generateVideo( }, }, }); - if (!result[historyId]) - throw new APIException(EX.API_IMAGE_GENERATION_FAILED, "记录不存在"); + if (!result[historyId]){ + logger.warn(`记录ID不存在： ${historyId} 重试次数: ${emptyCount}`); + emptyCount--; + if(emptyCount<=0){ + throw new APIException(EX.API_HISTORY_EMPTY, "记录不存在： " + JSON.stringify(result)); + }else{ + status = 20; + continue; + } + } status = result[historyId].status; failCode = result[historyId].fail_code; item_list = result[historyId].item_list; @@ -192,7 +201,7 @@ export async function generateVideo( if (failCode === '2038') throw new APIException(EX.API_CONTENT_FILTERED); else - throw new APIException(EX.API_IMAGE_GENERATION_FAILED); + throw new APIException(EX.API_VIDEO_GENERATION_FAILED); } // Assuming success if status is not 30 (failed) and not 20 (pending) // and item_list is populated. @@ -211,7 +220,675 @@ export async function generateVideo( // If no valid URLs but no explicit error thrown earlier, consider it a failure. // This could happen if item_list is empty or items don't have video_url. videoTaskCache.finishTask(task_id, -2); // Failure - throw new APIException(EX.API_IMAGE_GENERATION_FAILED, "视频生成未返回有效链接"); + throw new APIException(EX.API_VIDEO_GENERATION_FAILED, "视频生成未返回有效链接"); + } + return validVideoUrls; +} catch (error) { + videoTaskCache.finishTask(task_id, -2); // Failure due to exception + throw error; // Re-throw the error to be handled by the caller +} +} +//视频 提升分辨率 +export async function upgradeVideoResolution( + task_id: string, + { + targetVideoId = "", + targetHistoryId = "", + targetSubmitId = "", + originHistoryId = "", + originComponentList = [] + }: { + targetVideoId: string, + targetHistoryId: string, + targetSubmitId: string, + originHistoryId: string, + originComponentList: any[], + }, + refreshToken: string +) { + const videoTaskCache = VideoTaskCache.getInstance(); + videoTaskCache.startTask(task_id); + + try { + + const { totalCredit } = await getCredit(refreshToken); + if (totalCredit <= 0) + await receiveCredit(refreshToken); + + const componentId = util.uuid(); + const originSubmitId = util.uuid(); + //生成视频返回的 historyId item_list[0].video.video_id + let video_id = targetVideoId; + //生成视频返回的 historyId + let pre_historyId = targetHistoryId; + let origin_historyId = originHistoryId; + //生成视频返回的 historyId submit_id + let previewSubmitId = targetSubmitId; + //生成视频返回的 historyId draft_content.component_list[0] + + let origin_component_list_item:any = originComponentList.find((a)=>{ + return a.process_type == 1; + }); + originComponentList.sort((a,b)=>{ + return b.process_type - a.process_type; + }) + let pro_component_list_item:any = originComponentList[0]; + let origin_video_gen_inputs = origin_component_list_item.abilities.gen_video.text_to_video_params.video_gen_inputs[0]; + let prompt = origin_video_gen_inputs.prompt; + let first_frame_image = origin_video_gen_inputs.first_frame_image; + let width = first_frame_image.width; + let height = first_frame_image.height; + let duration = origin_video_gen_inputs.duration_ms; + let metrics_extra = JSON.stringify({ + promptSource: "upscale", + //生成视频返回的 historyId 19680709245698 + originId:origin_historyId, + originSubmitId: originSubmitId, + //返回的 task.first_frame_image 信息 + coverInfo: { + width: first_frame_image.width, + height: first_frame_image.height, + format: "", + imageUri: first_frame_image.image_uri, + imageUrl:first_frame_image.uri, + smartCropLoc: null, + coverUrlMap: {}, + }, + generateTimes: 0, + isDefaultSeed: 1, + previewId: pre_historyId, + //生成视频用的submit_id + previewSubmitId: previewSubmitId, + imageNameMapping: {}, + }); + const { aigc_data } = await request( + "post", + "/mweb/v1/aigc_draft/generate", + refreshToken, + { + params: { + babi_param: encodeURIComponent( + JSON.stringify({ + scenario: "image_video_generation", + feature_key: "text_to_video", + feature_entrance: "to_video", + feature_entrance_detail: "to_image-text_to_video", + }) + ), + }, + data: { + extend: { + m_video_commerce_info: { + resource_id: "generate_video", + resource_id_type: "str", + resource_sub_type: "aigc", + benefit_type: "video_upscale" + }, + root_model: pro_component_list_item.abilities.gen_video.text_to_video_params.model_req_key, + template_id: "", + history_option: {}, + }, + submit_id: util.uuid(), + metrics_extra: metrics_extra, + draft_content: JSON.stringify({ + type: "draft", + id: util.uuid(), + min_version: DRAFT_VERSION, + min_features: [], + is_from_tsn: true, + version: "3.2.2", + main_component_id: componentId, + //上一步生成视频任务返回的 historyId 中 draft_content的内容作为第一项 + component_list: [ + ...originComponentList, + { + type: "video_base_component", + id: componentId, + min_version: DRAFT_V_VERSION, + //上一步生成视频任务返回的 historyId 中 draft_content的内容的id + parent_id: pro_component_list_item.id, + metadata: { + type: "", + id: util.uuid(), + created_platform: 3, + created_platform_version: "", + created_time_in_ms: Date.now(), + created_did: "", + }, + generate_type: "gen_video", + aigc_mode: "workbench", + abilities: { + type: "", + id: util.uuid(), + gen_video:{ + type: "", + id: util.uuid(), + text_to_video_params:{ + type: "", + id: util.uuid(), + video_gen_inputs:[ + { + type: "", + id: util.uuid(), + min_version: DRAFT_V_VERSION, + prompt: prompt, + first_frame_image:{ + type: "image", + id: util.uuid(), + source_from: "upload", + platform_type: 1, + name: "", + image_uri: first_frame_image.image_uri, + width: first_frame_image.width, + height: first_frame_image.height, + format: "", + uri: first_frame_image.image_uri, + }, + lens_motion_type: "", + video_mode:2, + //上一步生成视频任务返回的 historyId 中 的video_id + vid: video_id, + fps:24, + duration_ms:duration, + v2v_opt: { + type: "", + id: util.uuid(), + min_version: "3.1.0", + super_resolution: { + type: "", + id: util.uuid(), + enable: true, + target_width: width*2, + target_height: height*2, + origin_width: width, + origin_height: height, + }, + }, + //上一步生成视频任务返回的 historyId + origin_history_id: pre_historyId, + } + ], + video_aspect_ratio:pro_component_list_item.abilities.gen_video.text_to_video_params.video_aspect_ratio, + model_req_key: pro_component_list_item.abilities.gen_video.text_to_video_params.model_req_key, + }, + scene: "super_resolution", + //上面生成的 metrics_extra + video_task_extra:metrics_extra, + video_ref_params: { + type: "", + id: util.uuid(), + generate_type: 0, + item_id: (7512653500000000000 + Date.now()), + origin_history_id: pre_historyId, + }, + }, + }, + process_type:pro_component_list_item.process_type+1, + }, + ], + }), + }, + } + ); + const historyId = aigc_data.history_record_id; + if (!historyId) + throw new APIException(EX.API_VIDEO_GENERATION_FAILED, "高清 记录ID不存在"); + let status = 20, failCode, item_list = []; + //https://jimeng.jianying.com/mweb/v1/get_history_by_ids? + // + let emptyCount = 30; + while (status === 20) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + const result = await request("post", "/mweb/v1/get_history_by_ids", refreshToken, { + data: { + history_ids: [historyId], + http_common_info: { + aid: Number(DEFAULT_ASSISTANT_ID), + }, + }, + }); + if (!result[historyId]){ + logger.warn(`高清 记录ID不存在： ${historyId} 重试次数: ${emptyCount}`); + emptyCount--; + if(emptyCount<=0){ + throw new APIException(EX.API_HISTORY_EMPTY, "高清 记录不存在： " + JSON.stringify(result)); + }else{ + status = 20; + continue; + } + } + status = result[historyId].status; + failCode = result[historyId].fail_code; + item_list = result[historyId].item_list; + } + if (status === 30) { + if (failCode === '2038') + throw new APIException(EX.API_CONTENT_FILTERED); + else + throw new APIException(EX.API_VIDEO_GENERATION_FAILED); + } + // Assuming success if status is not 30 (failed) and not 20 (pending) + // and item_list is populated. + // A more robust check might be needed depending on actual API behavior for success. + const videoUrls = item_list.map((item) => { + if(!item?.video?.transcoded_video?.origin?.video_url) + return null; + return item.video.transcoded_video.origin.video_url; + }); + + // Filter out nulls and check if any valid URL was generated + const validVideoUrls = videoUrls.filter(url => url !== null); + if (validVideoUrls.length > 0) { + videoTaskCache.finishTask(task_id, -1, validVideoUrls.join(",")); // Success + } else { + // If no valid URLs but no explicit error thrown earlier, consider it a failure. + // This could happen if item_list is empty or items don't have video_url. + videoTaskCache.finishTask(task_id, -2); // Failure + throw new APIException(EX.API_VIDEO_GENERATION_FAILED, "高清 视频生成未返回有效链接"); + } + return validVideoUrls; +} catch (error) { + videoTaskCache.finishTask(task_id, -2); // Failure due to exception + throw error; // Re-throw the error to be handled by the caller +} +} + +//视频 补帧 (未完成) +export async function upgradeVideoFrame( + _model: string, + task_id: string, + prompt: string, + { + width = 512, + height = 512, + imgURL = "", + duration = 5000, + }: { + width: number; + height: number; + imgURL: string; + duration: number; + }, + refreshToken: string +) { + const videoTaskCache = VideoTaskCache.getInstance(); + videoTaskCache.startTask(task_id); + + try { + if(!imgURL){ + throw new APIException(EX.API_REQUEST_PARAMS_INVALID); + return; + } + const model = getModel(_model); + logger.info(`使用模型: ${_model} ： ${model} 参考图片尺寸: ${width}x${height} 图片地址 ${imgURL} 持续时间: ${duration} 提示词: ${prompt}`); + + const { totalCredit } = await getCredit(refreshToken); + if (totalCredit <= 0) + await receiveCredit(refreshToken); + + const componentId = util.uuid(); + const originSubmitId = util.uuid(); + const { aigc_data } = await request( + "post", + "/mweb/v1/aigc_draft/generate", + refreshToken, + { + params: { + babi_param: encodeURIComponent( + JSON.stringify({ + scenario: "image_video_generation", + feature_key: "text_to_video", + feature_entrance: "to_video", + feature_entrance_detail: "to_image-text_to_video", + }) + ), + }, + data: { + extend: { + m_video_commerce_info: { + resource_id: "generate_video", + resource_id_type: "str", + resource_sub_type: "aigc", + benefit_type: "basic_video_operation_vgfm_v_three" + }, + root_model: model, + template_id: "", + history_option: {}, + }, + submit_id: util.uuid(), + metrics_extra: JSON.stringify({ + promptSource: "custom", + originSubmitId: originSubmitId, + isDefaultSeed: 1, + originTemplateId: "", + imageNameMapping: {}, + }), + draft_content: JSON.stringify({ + type: "draft", + id: util.uuid(), + min_version: DRAFT_VERSION, + min_features: [], + is_from_tsn: true, + version: "3.2.2", + main_component_id: componentId, + component_list: [ + { + type: "video_base_component", + id: componentId, + min_version: DRAFT_V_VERSION, + generate_type: "gen_video", + aigc_mode: "workbench", + metadata: { + type: "", + id: util.uuid(), + created_platform: 3, + created_platform_version: "", + created_time_in_ms: Date.now(), + created_did: "", + }, + abilities: { + type: "", + id: util.uuid(), + gen_video:{ + type: "", + id: util.uuid(), + text_to_video_params:{ + type: "", + id: util.uuid(), + video_gen_inputs:[ + { + type: "", + id: util.uuid(), + min_version: DRAFT_V_VERSION, + prompt: prompt, + first_frame_image:{ + type: "image", + id: util.uuid(), + source_from: "upload", + platform_type: 1, + name: "", + image_uri: imgURL, + width: width, + height: height, + format: "", + uri: imgURL, + }, + video_mode:2, + fps:24, + duration_ms:duration, + } + ], + video_aspect_ratio:"9:16", + seed: Math.floor(Math.random() * 100000000) + 2500000000, + model_req_key: model, + }, + video_task_extra:{ + promptSource: "custom", + originSubmitId: originSubmitId, + isDefaultSeed: 1, + originTemplateId: "", + imageNameMapping: {}, + } + }, + }, + process_type:1, + }, + ], + }), + http_common_info: { + aid: Number(DEFAULT_ASSISTANT_ID), + }, + }, + } + ); + const historyId = aigc_data.history_record_id; + if (!historyId) + throw new APIException(EX.API_VIDEO_GENERATION_FAILED, "记录ID不存在"); + let status = 20, failCode, item_list = []; + //https://jimeng.jianying.com/mweb/v1/get_history_by_ids? + // + while (status === 20) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + const result = await request("post", "/mweb/v1/get_history_by_ids", refreshToken, { + data: { + history_ids: [historyId], + http_common_info: { + aid: Number(DEFAULT_ASSISTANT_ID), + }, + }, + }); + if (!result[historyId]) + throw new APIException(EX.API_HISTORY_EMPTY, "记录不存在"); + status = result[historyId].status; + failCode = result[historyId].fail_code; + item_list = result[historyId].item_list; + } + if (status === 30) { + if (failCode === '2038') + throw new APIException(EX.API_CONTENT_FILTERED); + else + throw new APIException(EX.API_VIDEO_GENERATION_FAILED); + } + // Assuming success if status is not 30 (failed) and not 20 (pending) + // and item_list is populated. + // A more robust check might be needed depending on actual API behavior for success. + const videoUrls = item_list.map((item) => { + if(!item?.video?.transcoded_video?.origin?.video_url) + return null; + return item.video.transcoded_video.origin.video_url; + }); + + // Filter out nulls and check if any valid URL was generated + const validVideoUrls = videoUrls.filter(url => url !== null); + if (validVideoUrls.length > 0) { + videoTaskCache.finishTask(task_id, -1, validVideoUrls.join(",")); // Success + } else { + // If no valid URLs but no explicit error thrown earlier, consider it a failure. + // This could happen if item_list is empty or items don't have video_url. + videoTaskCache.finishTask(task_id, -2); // Failure + throw new APIException(EX.API_VIDEO_GENERATION_FAILED, "视频生成未返回有效链接"); + } + return validVideoUrls; +} catch (error) { + videoTaskCache.finishTask(task_id, -2); // Failure due to exception + throw error; // Re-throw the error to be handled by the caller +} +} + +//视频 生成音效 (未完成) +export async function generateVideoSound( + _model: string, + task_id: string, + prompt: string, + { + width = 512, + height = 512, + imgURL = "", + duration = 5000, + }: { + width: number; + height: number; + imgURL: string; + duration: number; + }, + refreshToken: string +) { + const videoTaskCache = VideoTaskCache.getInstance(); + videoTaskCache.startTask(task_id); + + try { + if(!imgURL){ + throw new APIException(EX.API_REQUEST_PARAMS_INVALID); + return; + } + const model = getModel(_model); + logger.info(`使用模型: ${_model} ： ${model} 参考图片尺寸: ${width}x${height} 图片地址 ${imgURL} 持续时间: ${duration} 提示词: ${prompt}`); + + const { totalCredit } = await getCredit(refreshToken); + if (totalCredit <= 0) + await receiveCredit(refreshToken); + + const componentId = util.uuid(); + const originSubmitId = util.uuid(); + const { aigc_data } = await request( + "post", + "/mweb/v1/aigc_draft/generate", + refreshToken, + { + params: { + babi_param: encodeURIComponent( + JSON.stringify({ + scenario: "image_video_generation", + feature_key: "text_to_video", + feature_entrance: "to_video", + feature_entrance_detail: "to_image-text_to_video", + }) + ), + }, + data: { + extend: { + m_video_commerce_info: { + resource_id: "generate_video", + resource_id_type: "str", + resource_sub_type: "aigc", + benefit_type: "basic_video_operation_vgfm_v_three" + }, + root_model: model, + template_id: "", + history_option: {}, + }, + submit_id: util.uuid(), + metrics_extra: JSON.stringify({ + promptSource: "custom", + originSubmitId: originSubmitId, + isDefaultSeed: 1, + originTemplateId: "", + imageNameMapping: {}, + }), + draft_content: JSON.stringify({ + type: "draft", + id: util.uuid(), + min_version: DRAFT_VERSION, + min_features: [], + is_from_tsn: true, + version: "3.2.2", + main_component_id: componentId, + component_list: [ + { + type: "video_base_component", + id: componentId, + min_version: DRAFT_V_VERSION, + generate_type: "gen_video", + aigc_mode: "workbench", + metadata: { + type: "", + id: util.uuid(), + created_platform: 3, + created_platform_version: "", + created_time_in_ms: Date.now(), + created_did: "", + }, + abilities: { + type: "", + id: util.uuid(), + gen_video:{ + type: "", + id: util.uuid(), + text_to_video_params:{ + type: "", + id: util.uuid(), + video_gen_inputs:[ + { + type: "", + id: util.uuid(), + min_version: DRAFT_V_VERSION, + prompt: prompt, + first_frame_image:{ + type: "image", + id: util.uuid(), + source_from: "upload", + platform_type: 1, + name: "", + image_uri: imgURL, + width: width, + height: height, + format: "", + uri: imgURL, + }, + video_mode:2, + fps:24, + duration_ms:duration, + } + ], + video_aspect_ratio:"9:16", + seed: Math.floor(Math.random() * 100000000) + 2500000000, + model_req_key: model, + }, + video_task_extra:{ + promptSource: "custom", + originSubmitId: originSubmitId, + isDefaultSeed: 1, + originTemplateId: "", + imageNameMapping: {}, + } + }, + }, + process_type:1, + }, + ], + }), + http_common_info: { + aid: Number(DEFAULT_ASSISTANT_ID), + }, + }, + } + ); + const historyId = aigc_data.history_record_id; + if (!historyId) + throw new APIException(EX.API_VIDEO_GENERATION_FAILED, "记录ID不存在"); + let status = 20, failCode, item_list = []; + //https://jimeng.jianying.com/mweb/v1/get_history_by_ids? + // + while (status === 20) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + const result = await request("post", "/mweb/v1/get_history_by_ids", refreshToken, { + data: { + history_ids: [historyId], + http_common_info: { + aid: Number(DEFAULT_ASSISTANT_ID), + }, + }, + }); + if (!result[historyId]) + throw new APIException(EX.API_HISTORY_EMPTY, "记录不存在"); + status = result[historyId].status; + failCode = result[historyId].fail_code; + item_list = result[historyId].item_list; + } + if (status === 30) { + if (failCode === '2038') + throw new APIException(EX.API_CONTENT_FILTERED); + else + throw new APIException(EX.API_VIDEO_GENERATION_FAILED); + } + // Assuming success if status is not 30 (failed) and not 20 (pending) + // and item_list is populated. + // A more robust check might be needed depending on actual API behavior for success. + const videoUrls = item_list.map((item) => { + if(!item?.video?.transcoded_video?.origin?.video_url) + return null; + return item.video.transcoded_video.origin.video_url; + }); + + // Filter out nulls and check if any valid URL was generated + const validVideoUrls = videoUrls.filter(url => url !== null); + if (validVideoUrls.length > 0) { + videoTaskCache.finishTask(task_id, -1, validVideoUrls.join(",")); // Success + } else { + // If no valid URLs but no explicit error thrown earlier, consider it a failure. + // This could happen if item_list is empty or items don't have video_url. + videoTaskCache.finishTask(task_id, -2); // Failure + throw new APIException(EX.API_VIDEO_GENERATION_FAILED, "视频生成未返回有效链接"); } return validVideoUrls; } catch (error) { @@ -222,4 +899,7 @@ export async function generateVideo( export default { generateVideo, + upgradeVideoResolution, + // upgradeVideoFrame, + // generateVideoSound, }; diff --git a/src/api/routes/video.ts b/src/api/routes/video.ts index ef5055f..32b2f16 100644 --- a/src/api/routes/video.ts +++ b/src/api/routes/video.ts @@ -1,7 +1,7 @@ import _ from "lodash"; import Request from "@/lib/request/Request.ts"; -import { generateVideo } from "@/api/controllers/video.ts"; +import { generateVideo, upgradeVideoResolution } from "@/api/controllers/video.ts"; import { tokenSplit } from "@/api/controllers/core.ts"; import util from "@/lib/util.ts"; import { VideoTaskCache } from '@/api/VideoTaskCache.ts'; @@ -73,5 +73,44 @@ export default { data:'success', }; }, + "/upscale": async (request: Request) => { + request + .validate("body.task_id", _.isString) + .validate("body.targetVideoId", _.isString) + .validate("body.targetHistoryId", _.isString) + .validate("body.targetSubmitId", _.isString) + .validate("body.originHistoryId", _.isString) + .validate("body.components", _.isString) + .validate("headers.authorization", _.isString); + // refresh_token切分 必须和generations使用同一个token + const tokens = tokenSplit(request.headers.authorization); + // 取第一个 必须和generations使用同一个token + const token = tokens[0]; + const { + task_id, + targetVideoId, + targetHistoryId, + targetSubmitId, + originHistoryId, + components, + } = request.body; + const originComponentList = JSON.parse(components); + //不等结果 直接返回 + upgradeVideoResolution(task_id, { + targetVideoId, + targetHistoryId, + targetSubmitId, + originHistoryId, + originComponentList + }, token); + // let data = []; + // data = imageUrls.map((url) => ({ + // url, + // })); + return { + created: util.unixTimestamp(), + data:'success', + }; + }, }, };