加入收藏 | 设为首页 | 会员中心 | 我要投稿 核心网 (https://www.hxwgxz.com/)- 科技、建站、经验、云计算、5G、大数据,站长网!
当前位置: 首页 > 大数据 > 正文

利用文本挖掘技术来找出网络中的“小鲜词”

发布时间:2021-01-20 10:15:34 所属栏目:大数据 来源:网络整理
导读:开始之前,先看一下从人人网中发现的90后用户爱用的词 是不是很好玩,哈哈。写这篇文章就是让你简单的自动的从文本中找出新的词,这样就知道现在的年轻人喜欢什么了(对于博主这种上了年纪的人来说,真的是很有用,呜呜) 项目结构 当然,text.dat和common.d

这里写图片描述

这几个类用于给词创建索引,方便从词典中找出

CnPreviewTextIndexer.java

package grid.text.index;

import grid.common.TextUtils;

import java.util.HashMap;
import java.util.Map;
import java.util.Vector;

public class CnPreviewTextIndexer implements TextIndexer {

    private final static int CN_LETTER_COUNT = 5021;

    private String document;

    private Map<Character,Vector<Integer>> posMap;

    public CnPreviewTextIndexer(String document) {
        this.document = document;
        init();
    }

    private void init() {
        final int len = document.length();

        final int supposedMinCount = 1 + (int) Math.log(len / CN_LETTER_COUNT
                + 1);

        char c;

        Vector<Integer> posVector;

        posMap = new HashMap<Character,Vector<Integer>>(CN_LETTER_COUNT);

        for (int i = 0; i < len; i++) {
            c = document.charAt(i);
            if (!TextUtils.isCnLetter(c)) {
                continue;
            }
            posVector = posMap.get(c);
            if (null == posVector) {
                posVector = new Vector<Integer>(supposedMinCount);
                posMap.put(c,posVector);
            }
            posVector.add(i);
        }
    }

    @Override
    public int count(String text) {

        if (TextUtils.isBlank(text)) {
            return 0;
        }

        Vector<Integer> vector = posMap.get(text.charAt(0));

        if (null == vector) {
            return 0;
        }

        if (1 == text.length()) {
            return vector.size();
        }

        final int size = vector.size();
        int count = 0;

        for (int i = 0; i < size; i++) {
            if (TextUtils.match(document,vector.get(i),text)) {
                count++;
            }
        }

        return count;
    }

    @Override
    public Pos find(Pos pos) {
        String text = pos.getTarget();

        pos.setFound(false);

        if (TextUtils.isBlank(text)) {
            return pos;
        }

        Vector<Integer> vector = posMap.get(text.charAt(0));

        if (null == vector) {
            return pos;
        }

        final int arraySize = vector.size();
        final int arrayIndex = pos.arrayIndex + 1;

        for (int i = arrayIndex; i < arraySize; i++) {
            if (TextUtils.match(document,text)) {
                pos.setFound(true);
                pos.setPos(vector.get(i));
                pos.arrayIndex = i;
                break;
            }
        }

        return pos;
    }

    @Override
    public int len() {
        return document.length();
    }

    @Override
    public String sub(int off,int len) {
        if (off < 0 || off + len >= document.length()) {
            return "";
        }
        return document.substring(off,off + len);
    }

    @Override
    public char charAt(int index) {
        if (index < 0 || index >= document.length()) {
            return 0;
        }
        return document.charAt(index);
    }
}

Pos.java

package grid.text.index;


public class Pos {
    private String target;

    /** * Pos for current matched full target text */
    private int pos = -1;

    /** * Index in position array for current matched full target text */
    int arrayIndex = -1;

    private boolean found = false;

    public Pos(String target) {
        this.target = target;
    }

    public String getTarget() {
        return target;
    }

    public int getPos() {
        return pos;
    }

    public boolean isFound() {
        return found;
    }

    void setPos(int pos) {
        this.pos = pos;
    }

    void setFound(boolean found) {
        this.found = found;
    }
}

SimpleTextIndexer.java

package grid.text.index;


public class SimpleTextIndexer implements TextIndexer {

    private String document;

    public SimpleTextIndexer(String document) {
        this.document = document;
    }

    @Override
    public int count(String text) {
        int off = 0;
        int count = 0;
        final int len = text.length();
        while ((off = document.indexOf(text,off)) > -1) {
            count++;
            off += len;
        }
        return count;
    }

    @Override
    public Pos find(Pos pos) {
        final String text = pos.getTarget();
        final int len = text.length();
        int off = pos.getPos() + len;
        if (pos.getPos() < 0)
            off = 0;

        pos.setFound(false);

        if ((off = document.indexOf(text,off)) > -1) {
            pos.setFound(true);
            pos.setPos(off);
        }
        return pos;
    }

    @Override
    public int len() {
        return document.length();
    }

    @Override
    public String sub(int off,int len) {
        return document.substring(off,off + len);
    }

    @Override
    public char charAt(int index) {
        if (index < 0 || index >= document.length()) {
            return 0;
        }
        return document.charAt(index);
    }
}

TextIndexer.java

package grid.text.index;


public interface TextIndexer {

    /** * @param text * @return count for specific text */
    public int count(String text);

    /** * @param pos * @return next position for current pos */
    public Pos find(Pos pos);

    /** * @return original document length */
    public int len();

    /** * @param off * @param len * @return the sub string start from <b>off</b> and with a length with * <b>len</b> */
    public String sub(int off,int len);

    /** * @param index * @return return the character in the specified index */
    public char charAt(int index);
}

participle

(编辑:核心网)

【声明】本站内容均来自网络,其相关言论仅代表作者个人观点,不代表本站立场。若无意侵犯到您的权利,请及时与联系站长删除相关内容!

热点阅读