数字人视频

康鲁杰

2025-03-24 b398796000c00386e056b8a5cd762f210abf4869

数字人视频

 yudao-module-digitalcourse/yudao-module-digitalcourse-biz/src/main/java/cn/iocoder/yudao/module/digitalcourse/controller/admin/digitalhumans/vo/DigitalHumansSaveReqVO.java

@@ -63,7 +63,6 @@
    private String useModel;

    @Schema(description = "状态(0: 正常, 1: 待审核，2：已受理，3：训练中，4：不通过，5：训练失败)", requiredMode = Schema.RequiredMode.REQUIRED, example = "2")
    @NotNull(message = "状态不能为空")
    private Integer status;

    //修复图片

 yudao-module-digitalcourse/yudao-module-digitalcourse-biz/src/main/java/cn/iocoder/yudao/module/digitalcourse/dal/dataobject/digitalhumans/DigitalHumansDO.java

@@ -132,4 +132,7 @@
     */
    private Date expireDate;

    private String referenceAudioText;
    private String asrFormatAudioUrl;

}

 yudao-module-digitalcourse/yudao-module-digitalcourse-biz/src/main/java/cn/iocoder/yudao/module/digitalcourse/service/digitalhumans/DigitalHumansServiceImpl.java

@@ -36,6 +36,7 @@
    @Override
    public Long createDigitalHumans(DigitalHumansSaveReqVO createReqVO) {
        createReqVO.setCode(UUID.fastUUID().toString());
        createReqVO.setStatus(1);
        // 插入
        DigitalHumansDO digitalHumans = BeanUtils.toBean(createReqVO, DigitalHumansDO.class);
        digitalHumansMapper.insert(digitalHumans);
@@ -61,7 +62,7 @@

        //异步训练模型
        if (updateObj.getStatus() == 3){
            digitalHumansServiceUtil.remoteTrain(transferVO(updateObj.getId()));
            digitalHumansServiceUtil.remoteHeyGemTrain(transferVO(updateObj.getId()));
        }
    }


 yudao-module-digitalcourse/yudao-module-digitalcourse-biz/src/main/java/cn/iocoder/yudao/module/digitalcourse/service/digitalhumans/DigitalHumansServiceUtil.java

@@ -6,6 +6,7 @@
import cn.iocoder.yudao.module.digitalcourse.dal.dataobject.digitalhumans.DigitalHumansDO;
import cn.iocoder.yudao.module.digitalcourse.dal.mysql.digitalhumans.DigitalHumansMapper;
import cn.iocoder.yudao.module.infra.api.config.ConfigApi;
import cn.iocoder.yudao.module.infra.api.file.FileApi;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
@@ -19,6 +20,14 @@
import org.springframework.validation.annotation.Validated;
import org.springframework.scheduling.annotation.Async;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
@@ -31,6 +40,11 @@
@Validated
public class DigitalHumansServiceUtil {

    private static final String HEYGEM_CORE_URL = "heygem.core.url";
    private static final String HEYGEM_VOICE_DATA = "heygem.voice.data";
    private static final String HEYGEM_FACE2FACE = "heygem.face2face";
    private static final String EASEGEN_URL = "easegen.url";

    private static final String EASEGEN_CORE_URL = "easegen.core.url";

    private static final String EASEGEN_CORE_KEY = "easegen.core.key";
@@ -41,9 +55,149 @@

    @Resource
    private DigitalHumansMapper digitalHumansMapper;

    @Resource
    private ConfigApi configApi;

    public void remoteHeyGemTrain(DigitalHumansTrailVO digitalHumansTrailVo){
        String origin_audio = configApi.getConfigValueByKey(HEYGEM_VOICE_DATA) + "/origin_audio";
        String temp = configApi.getConfigValueByKey(HEYGEM_FACE2FACE) + "/temp";
        //训练前校验
        try {
            Files.createDirectories(Path.of(origin_audio));
            Files.createDirectories(Path.of(temp));
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        //todo 视频抠图
        String extname = digitalHumansTrailVo.getFixVideoUrl().substring(digitalHumansTrailVo.getFixVideoUrl().lastIndexOf("."));
        String modelFileName = new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date()) + extname;
        String modelFilePath = Paths.get(temp, modelFileName).toString();

        String substring = configApi.getConfigValueByKey(EASEGEN_URL)+digitalHumansTrailVo.getFixVideoUrl().substring(digitalHumansTrailVo.getFixVideoUrl().lastIndexOf("/"));

        try {
            Files.copy(Path.of(substring), Path.of(modelFilePath), StandardCopyOption.REPLACE_EXISTING);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }


        // 音频文件路径
        String audioFileName = modelFileName.replace(extname, ".wav");
        String audioFilePath = Paths.get(origin_audio, audioFileName).toString();

        // 使用 FFmpeg 提取音频
        extractAudio(modelFilePath, audioFilePath);

        System.out.println("视频已存储: " + modelFilePath);
        System.out.println("音频已提取: " + audioFilePath);
        //audioFilePath切除configValueByKey
        String configValueByKey = configApi.getConfigValueByKey(HEYGEM_VOICE_DATA);
        // 计算相对路径
        Path relativeAudioPath = Path.of(configValueByKey).relativize(Path.of(audioFilePath));
        Map<String, Object> map = Map.of(
                "format", "wav",
                "reference_audio", relativeAudioPath.toString().replace("\\", "/"),
                "lang", "zh"
        );
        // 将路径中的\替换为/  relativeAudioPath.toString()

        int maxRetries = 3; // 最大重试次数
        int retryCount = 0;  // 当前重试次数
        boolean success = false;

        while (retryCount < maxRetries && !success) {
            try {
                // 发送POST请求
                HttpResponse execute = HttpRequest.post(configApi.getConfigValueByKey(HEYGEM_CORE_URL) + "/v1/preprocess_and_tran")
                        .body(JSON.toJSONString(map))
                        .execute();
                String body = execute.body();

                // 检查响应状态码是否成功
                if (execute.getStatus() != 200) {
                    retryCount++;
                    if (retryCount >= maxRetries) {
                        // 超过重试次数，训练失败
                        digitalHumansMapper.update(new UpdateWrapper<DigitalHumansDO>().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS));
                        log.error("训练失败：->>>>>>>>>", execute.getStatus());
                        return;
                    }
                    continue; // 重新尝试
                }
                retryCount++;
                if (retryCount >= maxRetries) {
                    digitalHumansMapper.update(new UpdateWrapper<DigitalHumansDO>().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS));
                    log.error("训练失败：->>>>>>>>>");
                    return;
                }
                // 解析响应，检查是否有错误信息
                JSONObject responseJson = JSON.parseObject(body);
                // 处理业务逻辑错误，更新状态和错误信息
                String referenceAudioText = responseJson.getString("reference_audio_text");
                String asrFormatAudioUrl = responseJson.getString("asr_format_audio_url");
                // 如果成功，更新状态为0（成功）

                digitalHumansMapper.update(
                        new UpdateWrapper<DigitalHumansDO>()
                                .lambda()
                                .eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode())  // 条件：code 等于传入的值
                                .set(DigitalHumansDO::getStatus, 0)  // 更新字段 status 为 0
                                .set(DigitalHumansDO::getAsrFormatAudioUrl,asrFormatAudioUrl)
                                .set(DigitalHumansDO::getReferenceAudioText,referenceAudioText)
                );
                success = true;
            }catch (Exception e){
                retryCount++;
                if (retryCount >= maxRetries) {
                    // 捕获异常，记录错误原因并更新状态
                    digitalHumansMapper.update(new UpdateWrapper<DigitalHumansDO>().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS));
                    log.error("训练失败：->>>>>>>>>", e.getMessage());
                    return;
                }

                try {
                    // 重试前等待一段时间，避免频繁请求
                    TimeUnit.SECONDS.sleep(2);
                } catch (InterruptedException ie) {
                    Thread.currentThread().interrupt(); // 处理中断异常
                    break;
                }
            }
        }

    }
    /**
     * 调用 FFmpeg 提取音频
     * @param videoFilePath 视频文件路径
     * @param audioFilePath 输出音频文件路径
     */
    private static void extractAudio(String videoFilePath, String audioFilePath) {
        try {
            ProcessBuilder builder = new ProcessBuilder(
                    "ffmpeg", "-i", videoFilePath, "-q:a", "0", "-map", "a", audioFilePath
            );
            builder.redirectErrorStream(true);
            Process process = builder.start();

            // 读取 FFmpeg 输出（可选）
            BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
            String line;
            while ((line = reader.readLine()) != null) {
                System.out.println(line);
            }

            int exitCode = process.waitFor();
            if (exitCode == 0) {
                System.out.println("音频提取成功: " + audioFilePath);
            } else {
                System.out.println("音频提取失败！");
            }
        } catch (IOException | InterruptedException e) {
            e.printStackTrace();
        }
    }
    public void remoteTrain(DigitalHumansTrailVO digitalHumansTrailVo){
        //训练前校验

@@ -112,7 +266,6 @@
        }

    }

    @Async
    public void queryRemoteTrainResult(){
        try {