日本語単語ベクトルchiVeを使ってみた―RNNで文章生成〈５〉

import numpy as np
import pickle

chive_path = 'C:/path/to/chive/nwjc.sudachi_full_abc_w2v.txt'

# npyとpickleの保存先
npy_path = 'save_data/emb_layer.npy'
pickle_path = 'save_data/word2id.pkl'


# データロード
def load_data():
    emb_layer = np.load(npy_path)
    with open(pickle_path, 'rb') as f:
        word2id = pickle.load(f)
    
    return emb_layer, word2id


emb_list = []
word2id = {}
vocab_size = 0

# つづきから行う場合は実行
# emb_layer, word2id = load_data()
# vocab_size = len(emb_layer)

# chiveのテキストデータを開く
with open(chive_path, 'r', encoding='utf-8') as chive_file:
    # 最初の行を読み込み(語彙数 次元数)
    line = chive_file.readline().strip('\n')
    print(line)
    word_num = int(line.split()[0])
    emb_size = int(line.split()[1])

    # 語彙数だけ繰り返し
    for i in range(word_num):
        line = chive_file.readline().strip('\n')

        # すでに読み込んだ行をスキップ
        if i < vocab_size:
            continue

        line_list = line.split()
        print(line_list[0], line_list[1])

        # 単語と単語IDを辞書へ
        word2id[line_list[0]] = i
        # 単語ベクトルをnp.float32で追加
        vec = np.array(line_list[-emb_size:], dtype=np.float32)
        emb_list.append(vec)

        # 10000ごとに保存
        if (i+1) % 10000 == 0:
            if i+1 == 10000:
                emb_layer = np.array(emb_list)
            else:
                # emb_listをemb_layerに結合
                emb_layer = np.concatenate((emb_layer, emb_list), axis=0)
            np.save(npy_path, emb_layer)
            emb_list = []

            with open(pickle_path, 'wb') as f:
                pickle.dump(word2id, f, protocol=-1)
            
            print('-'*50)
            print(i+1, '/', word_num, 'have been done')
            print('-'*50)

# すべて読み込んだら保存
emb_layer = np.concatenate((emb_layer, emb_list), axis=0)
np.save(npy_path, emb_layer)
with open(pickle_path, 'wb') as f:
    pickle.dump(word2id, f, protocol=-1)

print('complete!')

3644628 300

vec = emb_layer[word2id['君']]

import numpy as np
import pickle

npy_path = 'save_data/emb_layer.npy'
pickle_path = 'save_data/word2id.pkl'

# データロード
emb_layer = np.load(npy_path)
vocab_size = len(emb_layer)

with open(pickle_path, 'rb') as f:
    word2id = pickle.load(f)
    # 単語IDから単語を返す辞書も作成
    id2word = {v:k for k,v in word2id.items()}

# 語彙数を5万に絞る
short_vocab_size = 50000
emb_layer = emb_layer[:short_vocab_size]
for i in range(len(word2id) - short_vocab_size):
    word2id.pop(id2word[short_vocab_size + i])
    id2word.pop(short_vocab_size + i)
print('-'*50)


# コサイン類似度
def cos_similarity(x, y):
    nx = x / np.sqrt(np.sum(x**2))
    ny = y / np.sqrt(np.sum(y**2))
    return np.dot(nx, ny)


def most_similar(word, top=5):
    if word not in word2id:
        print(word + ' is not found')
        print('-'*50)
        return
    print(word + ' is the most similar to...')

    # 単語ベクトルを取得
    word_vec = emb_layer[word2id[word]]

    # コサイン類似度を計算
    similarity = np.zeros(short_vocab_size)
    for i in range(short_vocab_size):
        similarity[i] = cos_similarity(word_vec, emb_layer[i])

    # 上位を出力
    count = 0
    # 降順にするために-1をかける
    for word_id in (-1 * similarity).argsort():
        if count >= top:
            print('-'*50)
            return
        if id2word[word_id] == word:
            continue        
        print('%s: %s' % (id2word[word_id], similarity[word_id]))
        count += 1


# main
words = []
for word in words:
    most_similar(word)