beomi/kor2vec repository overview

pip install git+https://github.com/naver/kor2vec.git

kor2vec train -c corpus/path -o output/model.kor2vec

from kor2vec import Kor2Vec
kor2vec = Kor2Vec.load("../model/path")

kor2vec.embedding("안녕 아이오아이야 나는 클로바에서 왔어")
>>> torch.tensor(5, 128) # embedding vector

kor2vec.embedding("나는 도라에몽이라고 해 반가워", numpy=True)
>>> numpy.array(4, 128) # numpy embedding vector

input = kor2vec.to_seqs(["안녕 나는 뽀로로라고 해", "만나서 반가워 뽀로로"], seq_len=4)
kor2vec.forward(input)
>> torch.tensor([2, 4, 128])

from kor2vec import Kor2vec

kor2vec = Kor2Vec(embed_size=128)

kor2vec.train("../path/corpus", batch_size=128) # takes some time

kor2vec.save("../mode/path") # saving embedding

import torch.nn as nn
from kor2vec import Kor2vec

kor2vec = Kor2Vec.load("../model/path")
# or kor2vec = SejongVector()

lstm = nn.LSTM(128, 64, batch_first=True)
dense = nn.Linear(64, 1)

# Make tensor input
sentences = ["이 영화는 정말 대박이에요", "우와 진짜 재미있었어요"]

x = kor2vec.to_seqs(sentences, seq_len=10)
# >>> tensor(batch_size, seq_len, char_seq_len)

x = kor2vec(x) # tensor(batch_size, seq_len, 128)
_, (x, xc) = lstm(x) # tensor(batch_size, 64)
x = dense(x) # tensor(batch_size, 1)

Copyright 2018 NAVER Corp.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.