package com.yice.webadmin.app.util;

import com.github.houbb.opencc4j.util.ZhConverterUtil;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.seg.common.Term;
import com.yice.webadmin.app.constant.DatasetCleanConstant;
import com.yice.webadmin.app.constant.DatasetConstant;
import com.yice.webadmin.app.data.DatasetRule;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;

import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 数据清洗工具类
 */
@Slf4j
public class DataCleanerUtil {

    private static final Set<String> badWords = new HashSet<>();

    static {
        badWords.add("色情");
        badWords.add("淫秽");
        badWords.add("迷信");
        badWords.add("性行为");
        badWords.add("暴力");
    }

    /**
     * 定义清洗后的数据
     *
     * @param data 清洗数据
     * @param rules 清洗规则
     * @return 返回清洗后的数据
     */
    public static String buildCleanAfterData(String data, List<DatasetRule> rules) {
        if(StringUtils.isEmpty(data))  return DatasetConstant.EMPTY_STR;

        int count = 0;
        for (DatasetRule rule : rules) {
            if(rule.getArgs() > 0) {
                data = buildJsonData(rule.getName(), data, rule.getArgs());
                count ++ ;
            }
        }
        //表示没有清洗指标
        if(count == 0)  return DatasetConstant.EMPTY_STR;
        return data;
    }

    /**
     * 新增过滤条件
     * @param rule 规则
     * @param data 清洗数据
     * @param radio 阀值
     * @return 清洗后的数据
     */
    private static String buildJsonData(String rule, String data, double radio) {
        switch (rule) {
            case DatasetCleanConstant.REMOVE_INVISIBLE_CHARACTER:
                data = data.replaceAll("[\\x00-\\x1F\\x7F-\\x9F]", "");
                break;
            case DatasetCleanConstant.REPLACE_UNIFORM_WHITESPACE:
                data = data.replaceAll("(?<![a-zA-Z])[\\u0020\\u00A0\\u2000-\\u200A\\u2028\\u2029\\u3000](?![a-zA-Z])", "");
                break;
            case DatasetCleanConstant.REMOVE_NON_MEANING_CHARACTERS:
                data = data.replaceAll("[\\p{Cntrl}\\p{Cn}]", "");
                break;
            case DatasetCleanConstant.REPLACE_TRADITIONAL_CHINESE_TO_SIMPLIFIED:
                data = ZhConverterUtil.toSimple(data);
                break;
            case DatasetCleanConstant.REMOVE_WEB_IDENTIFIERS:
                data = data.replaceAll("<[^>]*>", "");
                break;
            case DatasetCleanConstant.REMOVE_EMOJI:
                data = data.replaceAll("[\\uE000-\\uF8FF]|\ud83c[\\ud000-\\udfff]|\ud83d[\\ud000-\\udfff]|\ud83e[\\ud000-\\udfff]", "");
                break;
            case DatasetCleanConstant.REPLACE_EMAILS:
                data = data.replaceAll("\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b", "EMAIL");
                break;
            case DatasetCleanConstant.REPLACE_IP:
                data = data.replaceAll("\\b(25[0-5]\\.|2[0-4][0-9]\\.|[01]?[0-9][0-9]?\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\b", "IP_ADDRESS");
                break;
            case DatasetCleanConstant.REPLACE_IDENTIFIER:
                data = data.replaceAll("\\d+", "");
                break;
            case DatasetCleanConstant.FILTER_CHECK_NUMBER_WORDS:
                data = filterNumberWords(data, 0, (int)radio);
                break;
            case DatasetCleanConstant.FILTER_CHECK_WORD_REPETITION_REMOVAL:
                data = filterWordRepetition(data, radio);
                break;
            case DatasetCleanConstant.FILTER_CHECK_CHARACTER_REPETITION_REMOVAL:
                data = filterCharacterRepetition(data, radio);
                break;
            case DatasetCleanConstant.FILTER_CHECK_SPECIAL_CHARACTERS:
                data = filterSpecialCharacters(data, radio);
                break;
            case DatasetCleanConstant.FILTER_CHECK_FLAGGED_WORDS:
                data = filterCheckFlaggedWords(data, radio);
                break;
        }
        return data;
    }

    public static void main(String [] args) {
        String data = "我可以提供各种类型的帮助，包括回答问题、提供信息、解决$#问题、提供建议等。只要是我能力范围内^*的需求，我都会尽力帮助用户解决。";
        data = filterSpecialCharacters(data, 0.2);
        System.out.println(data);
    }

    /**
     * 计算阀值
     * @param data 计算坏的数据
     * @return 返回阀值
     */
    private static double calculateBadWordRatio(String data) {
        // 标准分词
        List<Term> termList = HanLP.segment(data);
        int badWordsCount = 0;
        for (Term term : termList) {
            if (badWords.contains(term.word.toLowerCase())) {
                badWordsCount++;
            }
        }
        return badWordsCount / (double) termList.size();
    }

    /**
     * 检查文档的色情暴力词率
     * @param data 过滤数据
     * @param radio 阀值
     * @return 清洗后的数据
     */
    private static String filterCheckFlaggedWords(String data,double radio) {
        StringBuffer result = new StringBuffer();
        double badWordRatio = calculateBadWordRatio(data);
        if (badWordRatio > radio) {
            Pattern pattern = Pattern.compile("(" + String.join("|", badWords) + ")(?![\\w])");
            Matcher matcher = pattern.matcher(data);
            while (matcher.find()) {
                //如果出现关键字符，则直接替换为空白字符
                matcher.appendReplacement(result, "");
            }
            matcher.appendTail(result);
        } else {
            result.append(data);
        }
        return result.toString();
    }

    /**
     * 检查文档的词重复率
     * 计算词重复率并替换重复内容
     * @param document 文档内容字符串
     * @param threshold 阈值比率
     * @return 处理后的文档内容，如果词重复率超过阈值，则重复内容被替换为空字符
     */
    public static String filterCharacterRepetition(String document, double threshold) {
        // 使用HanLP进行分词
        List<Term> termList = HanLP.segment(document);

        // 统计每个词的出现次数
        Map<String, Integer> wordCountMap = new HashMap<>();
        for (Term term : termList) {
            String word = term.word;
            // 忽略空格
            if (StringUtils.isNotBlank(word)) {
                wordCountMap.put(word, wordCountMap.getOrDefault(word, 0) + 1);
            }
        }

        // 找到出现次数最多的词
        String mostFrequentWord = wordCountMap.entrySet().stream()
                .max(Comparator.comparingInt(Map.Entry::getValue))
                .map(Map.Entry::getKey)
                .orElse(null);

        // 如果没有词出现超过一次，直接返回原字符串
        if (mostFrequentWord == null) {
            return document;
        }

        // 计算最大词的重复率
        double repetitionRate = (double) (wordCountMap.get(mostFrequentWord) - 1) / (double) termList.size();

        return repetitionRate > threshold ? DatasetConstant.EMPTY_STR : document;
    }

    /**
     * 检查文档的字重复率
     * @param text 清洗数据
     * @param threshold 阀值
     * @return 清洗后的数据
     */
    public static String filterWordRepetition(String text, double threshold) {
        // 将文本转换为字符数组
        char[] characters = text.toCharArray();

        // 统计每个字的出现次数
        Map<Character, Integer> characterCountMap = new HashMap<>();
        for (char c : characters) {
            // 忽略空白符
            if (!Character.isWhitespace(c)) {
                characterCountMap.put(c, characterCountMap.getOrDefault(c, 0) + 1);
            }
        }

        // 找到出现次数最多的字
        Character mostFrequentCharacter = characterCountMap.entrySet().stream()
                .max(Comparator.comparingInt(Map.Entry::getValue))
                .map(Map.Entry::getKey)
                .orElse(null);

        // 如果没有字出现超过一次，直接返回原字符串
        if (mostFrequentCharacter == null) {
            return text;
        }

        // 计算最大字的重复率
        double repetitionRate = (double) (characterCountMap.get(mostFrequentCharacter) - 1) / (double) characters.length;

        return repetitionRate > threshold ? DatasetConstant.EMPTY_STR : text;
    }

    /**
     * 过滤检查文档的特殊字符率
     * @param data 过滤数据
     * @param radio 最大长度
     * @return 返回截取后的字符串
     */
    private static String filterSpecialCharacters(String data, double radio) {
        StringBuffer result = new StringBuffer();
        double specialCharacterRatio = calculateSpecialCharacterRatio(data);
        if (specialCharacterRatio <= radio) {
            result.append(data.replaceAll("[#$*^&()]", ""));
        } else {
            result.append(data);
        }
        return result.toString();
    }

    /**
     * 计算特殊字符的阀值
     * @param data 数据
     * @return 阀值
     */
    private static double calculateSpecialCharacterRatio(String data) {
        List<Term> termList = HanLP.segment(data);
        int specialCharactersCount = 0;

        for (Term term : termList) {
            if (term.word.matches(".*[#$%*^&()].*")) {
                specialCharactersCount++;
            }
        }
        return specialCharactersCount / (double) termList.size();
    }

    /**
     * 过滤检查文档的词数目
     * @param data 过滤数据
     * @param radio 最大词条目数据
     * @return 返回截取后的字符串
     */
    private static String filterNumberWords(String data, int radio) {
        data = data.toLowerCase();
        if (data.length() <= radio) {
            return data;
        }
        int startIndex = data.length() - radio;
        return data.substring(startIndex);
    }

    /**
     * 过滤检查文档的词数目
     * @param content 过滤数据
     * @param minWordCount 最小词条目数据
     * @param maxWordCount 最大词条目数据
     * @return 返回截取后的字符串
     */
    private static String filterNumberWords(String content, int minWordCount, int maxWordCount) {
        // 标准分词
        List<Term> termList = HanLP.segment(content);

        // 计算词的数量
        int wordCount = termList.size();

        // 检查词的数量是否在指定范围内
        if (wordCount < minWordCount || wordCount > maxWordCount) {
            // 如果词的数量不在范围内，返回空字符串
            return DatasetConstant.EMPTY_STR;
        }

        // 如果词的数量在范围内，返回原始文本
        return content;
    }
}
