package com.iden.bms.face; import cn.hutool.core.collection.CollUtil; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.face.monitor.FaceMonitor; import com.face.monitor.model.FaceModel; import com.iden.bms.service.PersonService; import com.iden.common.cache.RedisKeyConstant; import com.iden.common.cache.RedisUtil; import com.iden.common.entity.*; import com.iden.common.enums.HandleWarningStatusEnum; import com.iden.common.enums.PersonTypeEnum; import com.iden.common.service.IdenCameraService; import com.iden.common.service.IdenCameraVideoService; import com.iden.common.service.IdenCommunityService; import com.iden.common.service.IdenFaceImageService; import com.iden.common.util.ByteUtil; import com.iden.common.util.DateUtils; import com.iden.common.util.FileUtil; import com.iden.common.util.VideoUtil; import com.iden.common.vo.FaceRetrieveResultVO; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.io.File; import java.math.BigDecimal; import java.util.*; /** * @Author: stw * @Date: 2021/7/14 * @Desc: */ @Service public class FaceIdenService { @Resource private IdenFaceImageService idenFaceImageService; @Resource private IdenCameraService idenCameraService; @Resource private IdenCommunityService idenCommunityService; @Resource private PersonService personService; @Resource private IdenCameraVideoService idenCameraVideoService; @Resource private RedisUtil redisUtil; @Value("${iden.root:#{null}}") private String idenRoot; @Value("${file.url:#{null}}") private String fileUrl; private static final Logger logger = LogManager.getLogger(FaceIdenService.class); /** * 处理各摄像头上传的图像,去除没有人脸的、不是正面的、不符合标准的、连拍的图像 * @throws Exception */ public void handleCameraImage() throws Exception { logger.info("FaceIdenService.handleCameraImage start..."); File originImageDirFile = new File(idenRoot + "data/origin/camera/image"); //使用摄像头编码做目录名,里面图像的名称上加上拍照时间, // 比如 20211222123223_76639ced-6409-11ec-b8f9-fa163e4e1e9f.jpg if (originImageDirFile.isDirectory()) { File[] cameraCodeDirs = originImageDirFile.listFiles(); if(cameraCodeDirs != null && cameraCodeDirs.length > 0){ logger.info("FaceIdenService.handleCameraImage 1..."); //初始化引擎,加载人员图像人脸特征库 String personImageDir = idenRoot + "data/final/person/image"; //递归获取人员图像目录下的图像文件 File[] imgPersonFiles = FileUtil.getFiles(personImageDir); logger.info("FaceIdenService.handleCameraImage 2..."); FaceMonitor faceMonitorPerson = null; if(imgPersonFiles.length > 0){ faceMonitorPerson = FaceIdenTool.initFaceMonitor(idenRoot,imgPersonFiles); } try { for (File cameraCodedir : cameraCodeDirs) { if (cameraCodedir.isDirectory()) { File[] imgFiles = FileUtil.sortByName(cameraCodedir.listFiles()); if (imgFiles != null && imgFiles.length > 0) { String cameraCode = cameraCodedir.getName(); QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.lambda().eq(IdenCamera::getCode, cameraCode); IdenCamera idenCamera = idenCameraService.getOne(queryWrapper); if (idenCamera == null) { continue; } logger.info("FaceIdenService.handleCameraImage 3..."); //获取特征码 FaceModel[] faceModels = FaceIdenTool.extractFeature(idenRoot, imgFiles); logger.info("FaceIdenService.handleCameraImage 4..."); List idenFaceImageList = new ArrayList<>(); for (int i = 0; i < imgFiles.length; i++) { logger.info("FaceIdenService.handleCameraImage 41..."); File imgFile = imgFiles[i]; String imgFileName = imgFile.getName(); IdenFaceImage idenFaceImage = new IdenFaceImage(); //识别特征码 if (faceModels[i] != null) { logger.info("FaceIdenService.handleCameraImage 42..."); String featPtr = ByteUtil.byte2Hex(faceModels[i].getFeatValue()); logger.info("FaceIdenService.handleCameraImage 42...featPtr=="+featPtr); //没有人脸的、不是正面的、不符合标准的,连拍的图片 if (FaceIdenTool.isBad(featPtr) || isContinuation(cameraCode, imgFileName, featPtr)) { logger.info("FaceIdenService.handleCameraImage 43..."); File discardDir = new File(imgFile.getParentFile().getAbsolutePath().replace("origin", "discard")); if (!discardDir.exists()) { discardDir.mkdirs(); } File finalImgFile = new File(discardDir, imgFileName); imgFile.renameTo(finalImgFile);//移动到废弃目录 continue; } else { logger.info("FaceIdenService.handleCameraImage 44..."); idenFaceImage.setFeatPtr(featPtr); //存到redis redisUtil.hset(RedisKeyConstant.HANDLE_CAMERA_IMAGE_FIRST + "_" + cameraCode, imgFileName, featPtr, RedisKeyConstant.HANDLE_CAMERA_IMAGE_FIRST_TIME); } } logger.info("FaceIdenService.handleCameraImage 5..."); idenFaceImage.setCameraId(idenCamera.getId()); idenFaceImage.setCommunityId(idenCamera.getCommunityId()); String photographPlace = idenCamera.getPlace(); if (StringUtils.isEmpty(photographPlace)) { IdenCommunity idenCommunity = idenCommunityService.getById(idenCamera.getCommunityId()); if (idenCommunity != null) { photographPlace = idenCommunity.getName() + "(" + idenCamera.getLongitude() + ":" + idenCamera.getLatitude() + ")"; } else { photographPlace = "(" + idenCamera.getLongitude() + ":" + idenCamera.getLatitude() + ")"; } } idenFaceImage.setPhotographPlace(photographPlace); idenFaceImage.setLongitude(idenCamera.getLongitude()); idenFaceImage.setLatitude(idenCamera.getLatitude()); String photographTime = imgFileName.substring(0, imgFileName.indexOf("_")); idenFaceImage.setPhotographTime(DateUtils.strToDate(photographTime, "yyyyMMddHHmmss")); idenFaceImage.setDataDate(photographTime.substring(0, 8)); File finalDir = new File(imgFile.getParentFile().getAbsolutePath().replace("origin", "final")); if (!finalDir.exists()) { finalDir.mkdirs(); } logger.info("FaceIdenService.handleCameraImage ...finalDir == " + finalDir.getAbsolutePath()); logger.info("FaceIdenService.handleCameraImage ...imgFile.getName() == " + imgFileName); File finalImgFile = new File(finalDir, imgFileName); logger.info("FaceIdenService.handleCameraImage ...finalImgFile == " + finalImgFile.getAbsolutePath()); imgFile.renameTo(finalImgFile);//移动到最终目录 idenFaceImage.setImage(fileUrl + "camera/image/" + cameraCode + "/" + finalImgFile.getName()); //和人员图像库比对特征码,关联personId if (faceModels[i] != null && faceMonitorPerson != null) { FaceRetrieveResultVO faceRetrieveResultVO = FaceIdenTool.getHitResult(faceMonitorPerson, faceModels[i]); if (faceRetrieveResultVO != null) { int hitIndex = faceRetrieveResultVO.getIndex(); IdenPerson idenPerson = personService.getPersonIdByImageFileName(imgPersonFiles[hitIndex].getName()); if (idenPerson != null) { idenFaceImage.setPersonId(idenPerson.getId()); idenFaceImage.setUid(idenPerson.getUid()); idenFaceImage.setType(idenPerson.getType()); idenFaceImage.setGender(idenPerson.getGender()); float score = faceRetrieveResultVO.getScore(); BigDecimal b = new BigDecimal(score); double similarity = b.divide(new BigDecimal(100)).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue(); idenFaceImage.setSimilarity(similarity); } } } logger.info("FaceIdenService.handleCameraImage 6..."); if (idenFaceImage.getPersonId() == null) { //没有关联上,是陌生人 idenFaceImage.setType(PersonTypeEnum.STRANGER.getValue()); //查询以前的陌生人图库,进行人脸识别,若是同一个人,得到uid, 设置为一样 String uid = relevanceBeforeFace(idenFaceImage.getFeatPtr()); if (StringUtils.isEmpty(uid)) { uid = UUID.randomUUID().toString(); } idenFaceImage.setUid(uid); } idenFaceImage.setCreateTime(new Date()); idenFaceImageList.add(idenFaceImage); } logger.info("FaceIdenService.handleCameraImage 7..."); //把图像存到图库中 if (CollUtil.isNotEmpty(idenFaceImageList)) { idenFaceImageService.saveBatch(idenFaceImageList); } } } } } finally { //释放人脸引擎 FaceIdenTool.releaseEngine(faceMonitorPerson); } } } logger.info("FaceIdenService.handleCameraImage end"); } //查询以前的陌生人图库,进行人脸识别,若是同一个人,得到uid private String relevanceBeforeFace(String featPtrVisitor){ QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.lambda().eq(IdenFaceImage::getType, PersonTypeEnum.STRANGER.getValue()) .eq(IdenFaceImage::getHandleWarningStatus,HandleWarningStatusEnum.HANDLED.getValue()); //queryWrapper.last("limit 500"); //TODO 数据量大了需要分批处理 优化,以及是否区分小区 List list = this.idenFaceImageService.list(queryWrapper); if (CollUtil.isNotEmpty(list)) { List featPtrList = new ArrayList<>(); for(IdenFaceImage idenFaceImage : list) { String featPtr = idenFaceImage.getFeatPtr(); featPtrList.add(featPtr); } FaceRetrieveResultVO vo = FaceIdenTool.isHit(idenRoot,featPtrVisitor,featPtrList); if(vo != null) { return list.get(vo.getIndex()).getUid(); } } return null; } //是否连拍 private boolean isContinuation(String cameraCode,String imgFileName, String featPtr) { logger.info("FaceIdenService.isContinuation ...1"); String photographTime = imgFileName.substring(0, imgFileName.indexOf("_")); Long photographTimeLong = DateUtils.strToDate(photographTime,"yyyyMMddHHmmss").getTime(); //获取data/final/camera/image下图片 String originImageDir= idenRoot + "data/final/camera/image"; logger.info("FaceIdenService.isContinuation ...2"); File[] imgFiles = FileUtil.getFiles(originImageDir); List featPtrList = new ArrayList<>(); if (imgFiles != null && imgFiles.length > 0) { for(File imgFile : imgFiles) { String photographTimeTmp = imgFileName.substring(0, imgFile.getName().indexOf("_")); Long photographTimeTmpLong = DateUtils.strToDate(photographTimeTmp,"yyyyMMddHHmmss").getTime(); logger.info("FaceIdenService.isContinuation ...3"); //目标图片和以前图片拍照时间相差2分钟内 if (photographTimeLong <= photographTimeTmpLong + 2 * 60 * 1000) { String featPtrTmp = (String)redisUtil.hget(RedisKeyConstant.HANDLE_CAMERA_IMAGE_FIRST + "_" + cameraCode, imgFile.getName()); if(StringUtils.isNotEmpty(featPtrTmp)){ featPtrList.add(featPtrTmp); } } } logger.info("FaceIdenService.isContinuation ...4"); FaceRetrieveResultVO vo = FaceIdenTool.isHit(idenRoot,featPtr,featPtrList); logger.info("FaceIdenService.isContinuation ... 5==" + vo); return vo != null; } else { return false; } } /** * 把摄像头上传的图像解开为一帧帧图片,放到图像目录下,等待识别引擎处理 * @throws Exception */ public void handleCameraVideo() throws Exception { logger.info("FaceIdenService.handleCameraVideo start..."); File originVideoDirFile = new File(idenRoot + "data/origin/camera/video"); //使用摄像头编码做目录名,里面存放视频,名称里包含拍摄结束时间 // 比如 20211217123343_76639ced-6409-11ec-b8f9-fa883e4e1e9f.mp4 if (originVideoDirFile.isDirectory()) { File[] cameraCodeDirs = originVideoDirFile.listFiles(); if (cameraCodeDirs != null && cameraCodeDirs.length > 0) { for(File cameraCodedir : cameraCodeDirs) { if (cameraCodedir.isDirectory()) { File[] videoFiles = FileUtil.sortByName(cameraCodedir.listFiles()); if (videoFiles != null && videoFiles.length > 0) { String cameraCode = cameraCodedir.getName(); QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.lambda().eq(IdenCamera::getCode,cameraCode); IdenCamera idenCamera = idenCameraService.getOne(queryWrapper); if(idenCamera == null){ continue; } for(File videoFile : videoFiles){ String videoFileName = videoFile.getName(); String shootEndTimeStr = videoFileName.substring(0, videoFileName.indexOf("_")); Date shootEndTime = DateUtils.strToDate(shootEndTimeStr,"yyyyMMddHHmmss"); Long duration = VideoUtil.fetchAllPic(videoFile,idenRoot + "data/origin/camera/image/" + cameraCode, shootEndTime); File finalDir = new File(videoFile.getParentFile().getAbsolutePath().replace("origin","final")); if(!finalDir.exists()){ finalDir.mkdirs(); } logger.info("FaceIdenService.handleCameraVideo ...finalDir == " + finalDir.getAbsolutePath()); logger.info("FaceIdenService.handleCameraVideo ...videoFile.getName() == " + videoFileName); File finalvideoFile = new File(finalDir, videoFileName); logger.info("FaceIdenService.handleCameraVideo ...finalvideoFile == " + finalvideoFile.getAbsolutePath()); videoFile.renameTo(finalvideoFile);//移动到最终目录 IdenCameraVideo idenCameraVideo = new IdenCameraVideo(); idenCameraVideo.setVideoUrl(fileUrl + "camera/video/" + cameraCode + "/" + videoFileName); idenCameraVideo.setCameraId(idenCamera.getId()); idenCameraVideo.setCommunityId(idenCamera.getCommunityId()); idenCameraVideo.setName(videoFileName.substring(0,videoFileName.lastIndexOf(".mp4"))); Long begin = shootEndTime.getTime() - duration; idenCameraVideo.setPhotographBeginTime(new Date(begin)); idenCameraVideo.setPhotographEndTime(shootEndTime); idenCameraVideo.setDuration(duration); idenCameraVideo.setCreateTime(new Date()); idenCameraVideoService.save(idenCameraVideo); } } } } } } logger.info("FaceIdenService.handleCameraVideo end"); } }