私はword2vec
と優れたチュートリアルを使用してdoc2vec
、hereとhereを始めるしようとすると、コードサンプルを使用しようとしています。私はline_clean()
の句読点、ストップワードなどを削除する方法を追加しました。doc2vec/gensim - エポックでシャッフル文章の問題
しかし、私はトレーニングの反復で呼ばれるline_clean()
メソッドに問題があります。私はグローバルメソッドの呼び出しがそれを台無しにしていることを理解していますが、私はこの問題をどのように回避するか分からない。
Iteration 1
Traceback (most recent call last):
File "/Users/santino/Dev/doc2vec_exp/doc2vec_exp_app/doc2vec/untitled.py", line 96, in <module>
train()
File "/Users/santino/Dev/doc2vec_exp/doc2vec_exp_app/doc2vec/untitled.py", line 91, in train
model.train(sentences.sentences_perm(),total_examples=model.corpus_count,epochs=model.iter)
File "/Users/santino/Dev/doc2vec_exp/doc2vec_exp_app/doc2vec/untitled.py", line 61, in sentences_perm
shuffled = list(self.sentences)
AttributeError: 'TaggedLineSentence' object has no attribute 'sentences'
私のコードは以下の通りです:
import gensim
from gensim import utils
from gensim.models.doc2vec import TaggedDocument
from gensim.models import Doc2Vec
import os
import random
import numpy
from sklearn.linear_model import LogisticRegression
import logging
import sys
from nltk import RegexpTokenizer
from nltk.corpus import stopwords
tokenizer = RegexpTokenizer(r'\w+')
stopword_set = set(stopwords.words('english'))
def clean_line(line):
new_str = unicode(line, errors='replace').lower() #encoding issues
dlist = tokenizer.tokenize(new_str)
dlist = list(set(dlist).difference(stopword_set))
new_line = ' '.join(dlist)
return new_line
class TaggedLineSentence(object):
def __init__(self, sources):
self.sources = sources
flipped = {}
# make sure that keys are unique
for key, value in sources.items():
if value not in flipped:
flipped[value] = [key]
else:
raise Exception('Non-unique prefix encountered')
def __iter__(self):
for source, prefix in self.sources.items():
with utils.smart_open(source) as fin:
for item_no, line in enumerate(fin):
yield TaggedDocument(utils.to_unicode(clean_line(line)).split(), [prefix + '_%s' % item_no])
def to_array(self):
self.sentences = []
for source, prefix in self.sources.items():
with utils.smart_open(source) as fin:
for item_no, line in enumerate(fin):
self.sentences.append(TaggedDocument(utils.to_unicode(clean_line(line)).split(), [prefix + '_%s' % item_no]))
return(self.sentences)
def sentences_perm(self):
shuffled = list(self.sentences)
random.shuffle(shuffled)
return(shuffled)
def train():
#create a list data that stores the content of all text files in order of their names in docLabels
doc_files = [f for f in os.listdir('./data/') if f.endswith('.csv')]
sources = {}
for doc in doc_files:
doc2 = os.path.join('./data',doc)
sources[doc2] = doc.replace('.csv','')
sentences = TaggedLineSentence(sources)
# #iterator returned over all documents
model = gensim.models.Doc2Vec(size=300, min_count=2, alpha=0.025, min_alpha=0.025)
model.build_vocab(sentences)
#training of model
for epoch in range(10):
#random.shuffle(sentences)
print 'iteration '+str(epoch+1)
#model.train(it)
model.alpha -= 0.002
model.min_alpha = model.alpha
model.train(sentences.sentences_perm(),total_examples=model.corpus_count,epochs=model.iter)
#saving the created model
model.save('reddit.doc2vec')
print "model saved"
train()