package cn.iocoder.yudao.module.digitalcourse.service.digitalhumans; import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; import cn.iocoder.yudao.module.digitalcourse.controller.admin.digitalhumans.vo.DigitalHumansTrailVO; import cn.iocoder.yudao.module.digitalcourse.dal.dataobject.digitalhumans.DigitalHumansDO; import cn.iocoder.yudao.module.digitalcourse.dal.mysql.digitalhumans.DigitalHumansMapper; import cn.iocoder.yudao.module.infra.api.config.ConfigApi; import cn.iocoder.yudao.module.infra.api.file.FileApi; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; import org.springframework.scheduling.annotation.Async; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @Component @Validated public class DigitalHumansServiceUtil { private static final String HEYGEM_CORE_URL = "heygem.core.url"; private static final String HEYGEM_VOICE_DATA = "heygem.voice.data"; private static final String HEYGEM_FACE2FACE = "heygem.face2face"; private static final String EASEGEN_URL = "easegen.url"; private static final String EASEGEN_CORE_URL = "easegen.core.url"; private static final String EASEGEN_CORE_KEY = "easegen.core.key"; private static final int ERROR_STATUS = 5; private static final int COMPLETE_STATUS = 0; private static final int TARIN_STATUS = 3; @Resource private DigitalHumansMapper digitalHumansMapper; @Resource private ConfigApi configApi; @Resource private FileApi fileApi; @Async public void remoteHeyGemTrain(DigitalHumansTrailVO digitalHumansTrailVo){ String origin_audio = configApi.getConfigValueByKey(HEYGEM_VOICE_DATA) + "/origin_audio"; String temp = configApi.getConfigValueByKey(HEYGEM_FACE2FACE) + "/temp"; //训练前校验 try { Files.createDirectories(Path.of(origin_audio)); Files.createDirectories(Path.of(temp)); } catch (IOException e) { throw new RuntimeException(e); } //todo 视频抠图 String extname = digitalHumansTrailVo.getFixVideoUrl().substring(digitalHumansTrailVo.getFixVideoUrl().lastIndexOf(".")); String modelFileName = new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date()) + extname; String modelFilePath = Paths.get(temp, modelFileName).toString(); String substring = configApi.getConfigValueByKey(EASEGEN_URL)+digitalHumansTrailVo.getFixVideoUrl().substring(digitalHumansTrailVo.getFixVideoUrl().lastIndexOf("/")); try { Files.copy(Path.of(substring), Path.of(modelFilePath), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { throw new RuntimeException(e); } // 音频文件路径 String audioFileName = modelFileName.replace(extname, ".wav"); String audioFilePath = Paths.get(origin_audio, audioFileName).toString(); // 使用 FFmpeg 提取音频 extractAudio(modelFilePath, audioFilePath); System.out.println("视频已存储: " + modelFilePath); System.out.println("音频已提取: " + audioFilePath); //audioFilePath切除configValueByKey String configValueByKey = configApi.getConfigValueByKey(HEYGEM_VOICE_DATA); // 计算相对路径 Path relativeAudioPath = Path.of(configValueByKey).relativize(Path.of(audioFilePath)); Map map = Map.of( "format", "wav", "reference_audio", relativeAudioPath.toString().replace("\\", "/"), "lang", "zh" ); // 将路径中的\替换为/ relativeAudioPath.toString() int maxRetries = 3; // 最大重试次数 int retryCount = 0; // 当前重试次数 boolean success = false; while (retryCount < maxRetries && !success) { try { // 发送POST请求 HttpResponse execute = HttpRequest.post(configApi.getConfigValueByKey(HEYGEM_CORE_URL) + "/v1/preprocess_and_tran") .body(JSON.toJSONString(map)) .execute(); String body = execute.body(); // 检查响应状态码是否成功 if (execute.getStatus() != 200) { retryCount++; if (retryCount >= maxRetries) { // 超过重试次数,训练失败 digitalHumansMapper.update(new UpdateWrapper().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS)); log.error("训练失败:->>>>>>>>>", execute.getStatus()); return; } continue; // 重新尝试 } retryCount++; if (retryCount >= maxRetries) { digitalHumansMapper.update(new UpdateWrapper().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS)); log.error("训练失败:->>>>>>>>>"); return; } // 解析响应,检查是否有错误信息 JSONObject responseJson = JSON.parseObject(body); // 处理业务逻辑错误,更新状态和错误信息 String referenceAudioText = responseJson.getString("reference_audio_text"); String asrFormatAudioUrl = responseJson.getString("asr_format_audio_url"); // 如果成功,获取一帧当图片 String picFileName = modelFileName.replace(extname, ".png"); String pngPath = configApi.getConfigValueByKey(EASEGEN_URL) + "\\human_picture"; try { Files.createDirectories(Path.of(pngPath)); } catch (IOException e) { throw new RuntimeException(e); } String picFilePath = Paths.get(pngPath , picFileName).toString(); int frameTime = 3; // 截取第 5 秒的画面 extractFrame(modelFilePath, picFilePath, frameTime); byte[] fileContent = Files.readAllBytes(Paths.get(picFilePath)); // 读取文件内容 String fileUrl = fileApi.createFile(fileContent); System.out.println(fileUrl); digitalHumansMapper.update( new UpdateWrapper() .lambda() .eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()) // 条件:code 等于传入的值 .set(DigitalHumansDO::getStatus, 0) // 更新字段 status 为 0 .set(DigitalHumansDO::getAsrFormatAudioUrl,asrFormatAudioUrl) .set(DigitalHumansDO::getReferenceAudioText,referenceAudioText) .set(DigitalHumansDO::getPictureUrl,fileUrl) ); success = true; }catch (Exception e){ retryCount++; if (retryCount >= maxRetries) { // 捕获异常,记录错误原因并更新状态 digitalHumansMapper.update(new UpdateWrapper().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS)); log.error("训练失败:->>>>>>>>>", e.getMessage()); return; } try { // 重试前等待一段时间,避免频繁请求 TimeUnit.SECONDS.sleep(2); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); // 处理中断异常 break; } } } } /** * 调用 FFmpeg 提取音频 * @param videoFilePath 视频文件路径 * @param audioFilePath 输出音频文件路径 */ private static void extractAudio(String videoFilePath, String audioFilePath) { try { ProcessBuilder builder = new ProcessBuilder( "ffmpeg", "-i", videoFilePath, "-q:a", "0", "-map", "a", audioFilePath ); builder.redirectErrorStream(true); Process process = builder.start(); // 读取 FFmpeg 输出(可选) BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); String line; while ((line = reader.readLine()) != null) { System.out.println(line); } int exitCode = process.waitFor(); if (exitCode == 0) { System.out.println("音频提取成功: " + audioFilePath); } else { System.out.println("音频提取失败!"); } } catch (IOException | InterruptedException e) { e.printStackTrace(); } } /** * 调用 FFmpeg 提取视频帧 * @param videoPath 视频文件路径 * @param outputImagePath 输出图片路径 * @param frameTime 截取的时间点 */ public static void extractFrame(String videoPath, String outputImagePath, int frameTime) { ProcessBuilder processBuilder = new ProcessBuilder( "ffmpeg","-i", videoPath,"-ss", String.valueOf(frameTime), "-vframes", "1", "-q:v", "2", outputImagePath ); processBuilder.redirectErrorStream(true); // 合并错误输出 try { Process process = processBuilder.start(); BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); String line; while ((line = reader.readLine()) != null) { System.out.println(line); } int exitCode = process.waitFor(); // 等待执行完毕 if (exitCode == 0) { System.out.println("图片已成功提取到:" + outputImagePath); } else { System.out.println("FFmpeg 执行失败,错误代码:" + exitCode); } } catch (IOException | InterruptedException e) { System.out.println("FFmpeg 执行失败,发生异常:" + e.getMessage()); } } public void remoteTrain(DigitalHumansTrailVO digitalHumansTrailVo){ //训练前校验 int maxRetries = 3; // 最大重试次数 int retryCount = 0; // 当前重试次数 boolean success = false; ObjectMapper mapper = new ObjectMapper(); mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); while (retryCount < maxRetries && !success) { try { // 发送POST请求 HttpResponse execute = HttpRequest.post(configApi.getConfigValueByKey(EASEGEN_CORE_URL) + "/api/clone_digital_human") .header("X-API-Key", configApi.getConfigValueByKey(EASEGEN_CORE_KEY)) .body(mapper.writeValueAsString(digitalHumansTrailVo)) .execute(); String body = execute.body(); // 检查响应状态码是否成功 if (execute.getStatus() != 200) { retryCount++; if (retryCount >= maxRetries) { // 超过重试次数,训练失败 digitalHumansMapper.update(new UpdateWrapper().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS)); log.error("训练失败:->>>>>>>>>", execute.getStatus()); return; } continue; // 重新尝试 } // 解析响应,检查是否有错误信息 JSONObject responseJson = JSON.parseObject(body); if (!responseJson.getBoolean("success")) { // 处理业务逻辑错误,更新状态和错误信息 String errorDetail = responseJson.getString("detail"); retryCount++; if (retryCount >= maxRetries) { digitalHumansMapper.update(new UpdateWrapper().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS)); log.error("训练失败:->>>>>>>>>", errorDetail); return; } continue; // 重新尝试 } // 如果成功,更新状态为1(成功) // digitalHumansMapper.update(new UpdateWrapper().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, 0)); success = true; }catch (Exception e){ retryCount++; if (retryCount >= maxRetries) { // 捕获异常,记录错误原因并更新状态 digitalHumansMapper.update(new UpdateWrapper().lambda().eq(DigitalHumansDO::getCode, digitalHumansTrailVo.getCode()).set(DigitalHumansDO::getStatus, ERROR_STATUS)); log.error("训练失败:->>>>>>>>>", e.getMessage()); return; } try { // 重试前等待一段时间,避免频繁请求 TimeUnit.SECONDS.sleep(2); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); // 处理中断异常 break; } } } } @Async public void queryRemoteTrainResult(){ try { List codes = digitalHumansMapper.selectList(new QueryWrapper().lambda().eq(DigitalHumansDO::getStatus, TARIN_STATUS)) .stream().map(e -> e.getCode()).collect(Collectors.toList()); if (codes == null || codes.isEmpty()) return; // 批量调用远程接口 String result = HttpRequest.get(configApi.getConfigValueByKey(EASEGEN_CORE_URL) + "/api/clone_digital_human/result") .header("X-API-Key", configApi.getConfigValueByKey(EASEGEN_CORE_KEY)) .form("codes", String.join(",", codes)) .timeout(5000) // 设置超时时间 .execute() .body(); // 检查远程接口返回的结果是否有效 if (result == null || result.isEmpty()) { System.err.println("Remote API returned empty or null response."); return; } //打印结果 log.info("Remote API returned: " + result); if (JSON.isValidArray(result)) { JSONArray jsonArray = JSON.parseArray(result); Map resultMap = jsonArray.stream() .filter(obj -> JSON.isValidObject(JSON.toJSONString(obj))) .map(obj -> JSON.parseObject(JSON.toJSONString(obj))) .collect(Collectors.toMap(jsonObject -> jsonObject.getString("code"), jsonObject -> jsonObject)); log.info(JSON.toJSONString(resultMap)); Calendar instance = Calendar.getInstance(); instance.setTime(new Date()); instance.add(Calendar.YEAR, 1); Date time = instance.getTime(); codes.stream().forEach(e->{ JSONObject jsonObject = resultMap.get(e); if (jsonObject != null) { Integer status = jsonObject.getInteger("status"); if (status != null) { // 合并状态,0:训练成功,1:未开始,2:训练中,3:训练失败 if (status == 0){ String snapshotUrl = jsonObject.getString("snapshot_url"); digitalHumansMapper.update(new UpdateWrapper().lambda() .set(DigitalHumansDO::getStatus,COMPLETE_STATUS) .set(DigitalHumansDO::getExpireDate, time) .set(DigitalHumansDO::getSnapshotUrl, snapshotUrl) .set(DigitalHumansDO::getPictureUrl, snapshotUrl) .eq(DigitalHumansDO::getCode,e)); } else if (status == 3) { digitalHumansMapper.update(new UpdateWrapper().lambda().set(DigitalHumansDO::getStatus,ERROR_STATUS).eq(DigitalHumansDO::getCode,e)); } } else { log.error("Status is null for humans code: " + e); } }else { //如果没有匹配的记录,也修改为训练失败 digitalHumansMapper.update(new UpdateWrapper().lambda().set(DigitalHumansDO::getStatus,ERROR_STATUS).eq(DigitalHumansDO::getCode,e)); log.error("No matching result found for humans code: " + e); } }); }else { log.error("Invalid JSON array received from the remote API."); } }catch (Exception e){ // 捕获所有异常,防止程序崩溃 log.error("An error occurred while querying remote humans train result: " + e.getMessage()); e.printStackTrace(); } } }