iden_server/src/main/java/com/iden/bms/face/FaceIdenService.java

package com.iden.bms.face;




import cn.hutool.core.collection.CollUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.face.monitor.FaceMonitor;
import com.face.monitor.model.FaceModel;
import com.iden.bms.service.PersonService;
import com.iden.common.cache.RedisKeyConstant;
import com.iden.common.cache.RedisUtil;
import com.iden.common.entity.*;

import com.iden.common.enums.HandleWarningStatusEnum;
import com.iden.common.enums.PersonTypeEnum;
import com.iden.common.service.IdenCameraService;
import com.iden.common.service.IdenCameraVideoService;
import com.iden.common.service.IdenCommunityService;
import com.iden.common.service.IdenFaceImageService;

import com.iden.common.util.ByteUtil;
import com.iden.common.util.DateUtils;
import com.iden.common.util.FileUtil;
import com.iden.common.util.VideoUtil;
import com.iden.common.vo.FaceRetrieveResultVO;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.File;
import java.math.BigDecimal;
import java.util.*;


/**
 * @Author: stw
 * @Date: 2021/7/14
 * @Desc:
 */
@Service
public class FaceIdenService {
    @Resource
    private IdenFaceImageService idenFaceImageService;
    @Resource
    private IdenCameraService idenCameraService;
    @Resource
    private IdenCommunityService idenCommunityService;
    @Resource
    private PersonService personService;
    @Resource
    private IdenCameraVideoService idenCameraVideoService;

    @Resource
    private RedisUtil redisUtil;

    @Value("${iden.root:#{null}}")
    private String idenRoot;
    @Value("${file.url:#{null}}")
    private String fileUrl;

    private static final Logger logger = LogManager.getLogger(FaceIdenService.class);

    /**
     *  处理各摄像头上传的图像，去除没有人脸的、不是正面的、不符合标准的、连拍的图像
     * @throws Exception
     */
    public void handleCameraImage() throws Exception {
        logger.info("FaceIdenService.handleCameraImage start...");

        File originImageDirFile = new File(idenRoot + "data/origin/camera/image");

        //使用摄像头编码做目录名，里面图像的名称上加上拍照时间，
        // 比如  20211222123223_76639ced-6409-11ec-b8f9-fa163e4e1e9f.jpg
        if (originImageDirFile.isDirectory()) {
            File[] cameraCodeDirs = originImageDirFile.listFiles();
            if(cameraCodeDirs != null && cameraCodeDirs.length > 0){
                logger.info("FaceIdenService.handleCameraImage 1...");
                //初始化引擎，加载人员图像人脸特征库
                String personImageDir = idenRoot + "data/final/person/image";

                //递归获取人员图像目录下的图像文件
                File[] imgPersonFiles = FileUtil.getFiles(personImageDir);

                logger.info("FaceIdenService.handleCameraImage 2...");
                FaceMonitor faceMonitorPerson = null;
                if(imgPersonFiles.length > 0){
                    faceMonitorPerson = FaceIdenTool.initFaceMonitor(idenRoot,imgPersonFiles);
                }

                try {
                    for (File cameraCodedir : cameraCodeDirs) {
                        if (cameraCodedir.isDirectory()) {
                            File[] imgFiles = FileUtil.sortByName(cameraCodedir.listFiles());
                            if (imgFiles != null && imgFiles.length > 0) {

                                String cameraCode = cameraCodedir.getName();
                                QueryWrapper<IdenCamera> queryWrapper = new QueryWrapper<>();
                                queryWrapper.lambda().eq(IdenCamera::getCode, cameraCode);
                                IdenCamera idenCamera = idenCameraService.getOne(queryWrapper);
                                if (idenCamera == null) {
                                    continue;
                                }

                                logger.info("FaceIdenService.handleCameraImage 3...");
                                //获取特征码
                                FaceModel[] faceModels = FaceIdenTool.extractFeature(idenRoot, imgFiles);
                                logger.info("FaceIdenService.handleCameraImage 4...");
                                List<IdenFaceImage> idenFaceImageList = new ArrayList<>();
                                for (int i = 0; i < imgFiles.length; i++) {
                                    logger.info("FaceIdenService.handleCameraImage 41...");
                                    File imgFile = imgFiles[i];
                                    String imgFileName = imgFile.getName();
                                    IdenFaceImage idenFaceImage = new IdenFaceImage();
                                    //识别特征码
                                    if (faceModels[i] != null) {
                                        logger.info("FaceIdenService.handleCameraImage 42...");
                                        String featPtr = ByteUtil.byte2Hex(faceModels[i].getFeatValue());
                                        logger.info("FaceIdenService.handleCameraImage 42...featPtr=="+featPtr);
                                        //没有人脸的、不是正面的、不符合标准的,连拍的图片
                                        if (FaceIdenTool.isBad(featPtr) || isContinuation(cameraCode, imgFileName, featPtr)) {
                                            logger.info("FaceIdenService.handleCameraImage 43...");
                                            File discardDir = new File(imgFile.getParentFile().getAbsolutePath().replace("origin", "discard"));
                                            if (!discardDir.exists()) {
                                                discardDir.mkdirs();
                                            }
                                            File finalImgFile = new File(discardDir, imgFileName);
                                            imgFile.renameTo(finalImgFile);//移动到废弃目录
                                            continue;
                                        } else {
                                            logger.info("FaceIdenService.handleCameraImage 44...");
                                            idenFaceImage.setFeatPtr(featPtr);
                                            //存到redis
                                            redisUtil.hset(RedisKeyConstant.HANDLE_CAMERA_IMAGE_FIRST + "_" + cameraCode, imgFileName, featPtr, RedisKeyConstant.HANDLE_CAMERA_IMAGE_FIRST_TIME);
                                        }

                                    }
                                    logger.info("FaceIdenService.handleCameraImage 5...");
                                    idenFaceImage.setCameraId(idenCamera.getId());
                                    idenFaceImage.setCommunityId(idenCamera.getCommunityId());
                                    String photographPlace = idenCamera.getPlace();
                                    if (StringUtils.isEmpty(photographPlace)) {
                                        IdenCommunity idenCommunity = idenCommunityService.getById(idenCamera.getCommunityId());
                                        if (idenCommunity != null) {
                                            photographPlace = idenCommunity.getName() + "(" + idenCamera.getLongitude() + ":" + idenCamera.getLatitude() + ")";
                                        } else {
                                            photographPlace = "(" + idenCamera.getLongitude() + ":" + idenCamera.getLatitude() + ")";
                                        }
                                    }
                                    idenFaceImage.setPhotographPlace(photographPlace);
                                    idenFaceImage.setLongitude(idenCamera.getLongitude());
                                    idenFaceImage.setLatitude(idenCamera.getLatitude());

                                    String photographTime = imgFileName.substring(0, imgFileName.indexOf("_"));
                                    idenFaceImage.setPhotographTime(DateUtils.strToDate(photographTime, "yyyyMMddHHmmss"));
                                    idenFaceImage.setDataDate(photographTime.substring(0, 8));

                                    File finalDir = new File(imgFile.getParentFile().getAbsolutePath().replace("origin", "final"));
                                    if (!finalDir.exists()) {
                                        finalDir.mkdirs();
                                    }
                                    logger.info("FaceIdenService.handleCameraImage ...finalDir == " + finalDir.getAbsolutePath());
                                    logger.info("FaceIdenService.handleCameraImage ...imgFile.getName() == " + imgFileName);
                                    File finalImgFile = new File(finalDir, imgFileName);

                                    logger.info("FaceIdenService.handleCameraImage ...finalImgFile == " + finalImgFile.getAbsolutePath());

                                    imgFile.renameTo(finalImgFile);//移动到最终目录

                                    idenFaceImage.setImage(fileUrl + "camera/image/" + cameraCode + "/" + finalImgFile.getName());

                                    //和人员图像库比对特征码，关联personId
                                    if (faceModels[i] != null && faceMonitorPerson != null) {

                                        FaceRetrieveResultVO faceRetrieveResultVO = FaceIdenTool.getHitResult(faceMonitorPerson, faceModels[i]);
                                        if (faceRetrieveResultVO != null) {
                                            int hitIndex = faceRetrieveResultVO.getIndex();
                                            IdenPerson idenPerson = personService.getPersonIdByImageFileName(imgPersonFiles[hitIndex].getName());
                                            if (idenPerson != null) {
                                                idenFaceImage.setPersonId(idenPerson.getId());
                                                idenFaceImage.setUid(idenPerson.getUid());
                                                idenFaceImage.setType(idenPerson.getType());
                                                idenFaceImage.setGender(idenPerson.getGender());
                                                float score = faceRetrieveResultVO.getScore();
                                                BigDecimal b = new BigDecimal(score);
                                                double similarity = b.divide(new BigDecimal(100)).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue();

                                                idenFaceImage.setSimilarity(similarity);
                                            }
                                        }

                                    }
                                    logger.info("FaceIdenService.handleCameraImage 6...");
                                    if (idenFaceImage.getPersonId() == null) { //没有关联上，是陌生人
                                        idenFaceImage.setType(PersonTypeEnum.STRANGER.getValue());

                                        //查询以前的陌生人图库，进行人脸识别，若是同一个人，得到uid, 设置为一样
                                        String uid = relevanceBeforeFace(idenFaceImage.getFeatPtr());
                                        if (StringUtils.isEmpty(uid)) {
                                            uid = UUID.randomUUID().toString();
                                        }
                                        idenFaceImage.setUid(uid);
                                    }
                                    idenFaceImage.setCreateTime(new Date());
                                    idenFaceImageList.add(idenFaceImage);
                                }
                                logger.info("FaceIdenService.handleCameraImage 7...");
                                //把图像存到图库中
                                if (CollUtil.isNotEmpty(idenFaceImageList)) {
                                    idenFaceImageService.saveBatch(idenFaceImageList);
                                }

                            }
                        }
                    }
                } finally {
                    //释放人脸引擎
                    FaceIdenTool.releaseEngine(faceMonitorPerson);
                }

            }
        }
        logger.info("FaceIdenService.handleCameraImage end");
    }

    //查询以前的陌生人图库，进行人脸识别，若是同一个人，得到uid
    private String relevanceBeforeFace(String featPtrVisitor){
        QueryWrapper<IdenFaceImage> queryWrapper = new QueryWrapper<>();
        queryWrapper.lambda().eq(IdenFaceImage::getType, PersonTypeEnum.STRANGER.getValue())
                .eq(IdenFaceImage::getHandleWarningStatus,HandleWarningStatusEnum.HANDLED.getValue());
        //queryWrapper.last("limit 500"); //TODO 数据量大了需要分批处理 优化,以及是否区分小区
        List<IdenFaceImage> list = this.idenFaceImageService.list(queryWrapper);
        if (CollUtil.isNotEmpty(list)) {
            List<String> featPtrList = new ArrayList<>();
            for(IdenFaceImage idenFaceImage : list) {
                String featPtr = idenFaceImage.getFeatPtr();
                featPtrList.add(featPtr);
            }

            FaceRetrieveResultVO vo = FaceIdenTool.isHit(idenRoot,featPtrVisitor,featPtrList);
            if(vo != null) {
                return list.get(vo.getIndex()).getUid();
            }
        }

        return null;

    }


   //是否连拍
    private boolean isContinuation(String cameraCode,String imgFileName, String featPtr) {
        logger.info("FaceIdenService.isContinuation ...1");
        String photographTime = imgFileName.substring(0, imgFileName.indexOf("_"));
        Long photographTimeLong = DateUtils.strToDate(photographTime,"yyyyMMddHHmmss").getTime();
        //获取data/final/camera/image下图片
        String originImageDir= idenRoot + "data/final/camera/image";
        logger.info("FaceIdenService.isContinuation ...2");
        File[] imgFiles =  FileUtil.getFiles(originImageDir);
         List<String> featPtrList = new ArrayList<>();
        if (imgFiles != null && imgFiles.length > 0) {
            for(File imgFile : imgFiles) {
                String photographTimeTmp = imgFileName.substring(0, imgFile.getName().indexOf("_"));
                Long photographTimeTmpLong = DateUtils.strToDate(photographTimeTmp,"yyyyMMddHHmmss").getTime();
                logger.info("FaceIdenService.isContinuation ...3");
                //目标图片和以前图片拍照时间相差2分钟内
                if (photographTimeLong <= photographTimeTmpLong + 2 * 60 * 1000) {
                    String featPtrTmp = (String)redisUtil.hget(RedisKeyConstant.HANDLE_CAMERA_IMAGE_FIRST + "_" + cameraCode, imgFile.getName());
                    if(StringUtils.isNotEmpty(featPtrTmp)){
                        featPtrList.add(featPtrTmp);
                    }
                }
            }
            logger.info("FaceIdenService.isContinuation ...4");
            FaceRetrieveResultVO vo = FaceIdenTool.isHit(idenRoot,featPtr,featPtrList);
            logger.info("FaceIdenService.isContinuation ... 5==" + vo);
            return vo != null;

        } else {
            return false;
        }

    }
    /**
     * 把摄像头上传的图像解开为一帧帧图片，放到图像目录下，等待识别引擎处理
     * @throws Exception
     */
    public void handleCameraVideo() throws Exception {
        logger.info("FaceIdenService.handleCameraVideo start...");
        File originVideoDirFile = new File(idenRoot + "data/origin/camera/video");
        //使用摄像头编码做目录名，里面存放视频，名称里包含拍摄结束时间
        // 比如 20211217123343_76639ced-6409-11ec-b8f9-fa883e4e1e9f.mp4
        if (originVideoDirFile.isDirectory()) {
            File[] cameraCodeDirs = originVideoDirFile.listFiles();
            if (cameraCodeDirs != null && cameraCodeDirs.length > 0) {
                for(File cameraCodedir : cameraCodeDirs) {
                    if (cameraCodedir.isDirectory()) {
                        File[] videoFiles = FileUtil.sortByName(cameraCodedir.listFiles());
                        if (videoFiles != null && videoFiles.length > 0) {
                            String cameraCode = cameraCodedir.getName();
                            QueryWrapper<IdenCamera> queryWrapper = new QueryWrapper<>();
                            queryWrapper.lambda().eq(IdenCamera::getCode,cameraCode);
                            IdenCamera idenCamera = idenCameraService.getOne(queryWrapper);
                            if(idenCamera == null){
                                continue;
                            }

                            for(File videoFile : videoFiles){
                                String videoFileName = videoFile.getName();
                                String shootEndTimeStr = videoFileName.substring(0, videoFileName.indexOf("_"));
                                Date shootEndTime = DateUtils.strToDate(shootEndTimeStr,"yyyyMMddHHmmss");
                                Long duration = VideoUtil.fetchAllPic(videoFile,idenRoot + "data/origin/camera/image/" + cameraCode, shootEndTime);

                                File finalDir = new File(videoFile.getParentFile().getAbsolutePath().replace("origin","final"));
                                if(!finalDir.exists()){
                                    finalDir.mkdirs();
                                }
                                logger.info("FaceIdenService.handleCameraVideo ...finalDir == " + finalDir.getAbsolutePath());
                                logger.info("FaceIdenService.handleCameraVideo ...videoFile.getName() == " + videoFileName);
                                File finalvideoFile = new File(finalDir, videoFileName);

                                logger.info("FaceIdenService.handleCameraVideo ...finalvideoFile == " + finalvideoFile.getAbsolutePath());
                                videoFile.renameTo(finalvideoFile);//移动到最终目录

                                IdenCameraVideo idenCameraVideo = new IdenCameraVideo();
                                idenCameraVideo.setVideoUrl(fileUrl + "camera/video/" + cameraCode + "/" + videoFileName);
                                idenCameraVideo.setCameraId(idenCamera.getId());
                                idenCameraVideo.setCommunityId(idenCamera.getCommunityId());
                                idenCameraVideo.setName(videoFileName.substring(0,videoFileName.lastIndexOf(".mp4")));
                                Long begin = shootEndTime.getTime() - duration;
                                idenCameraVideo.setPhotographBeginTime(new Date(begin));
                                idenCameraVideo.setPhotographEndTime(shootEndTime);
                                idenCameraVideo.setDuration(duration);
                                idenCameraVideo.setCreateTime(new Date());
                                idenCameraVideoService.save(idenCameraVideo);
                            }
                        }
                    }
                }
            }
        }

        logger.info("FaceIdenService.handleCameraVideo end");
    }

}