Click here to Skip to main content
15,887,812 members
Please Sign up or sign in to vote.
0.00/5 (No votes)
See more: , +
i have a code
and i have this error:
ValueError:Empty vocabulary:perhaps document has only contain stop words.



this is stop words.txt:
a an the of as to in if

What I have tried:

#Created on Sat May 07 21:12:41 2016


from sklearn import cross_validation
from sklearn.datasets import load_files
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from nltk import wordpunct_tokenize, PorterStemmer, LancasterStemmer
import os
import nltk, os.path ,sklearn
#path = os.path.abspath(D:\\IR\\train)
#fi = load(path)
count_vect = CountVectorizer()    
tfidf_transformer = TfidfTransformer()    
clf= MultinomialNB()

data = load_files(container_path='D:\\train')
stopWordsFile = open('D:\\IR\\IR-Project\\StopWord.txt','r')
stopWords = stopWordsFile.read().split(' ')
porter = PorterStemmer()

#min_df=1

def train():    
    print ("training")
    for index in xrange(len(data.data)):
        tempData = data.data[index]
        tokens =  wordpunct_tokenize(tempData)
        #stopWordsRemoved = [unicode(token, 'utf-8') for token in tokens if token not in stopWords]
        stopWordsRemoved = [token.decode("windows-1252") for token in tokens if token not in stopWords]
        stems = [porter.stem(word) for word in stopWordsRemoved]
        data.data[index] = str(stems)
        
    
        
    trainSpars = count_vect.fit_transform(data.data)
    trainTfidf = tfidf_transformer.fit_transform(trainSpars)
    print (trainTfidf.shape)
    clf.fit(trainTfidf,data.target)

def evaluate(testSentence):  
    print ("evaluating")
    tokens =  wordpunct_tokenize(testSentence)
    stopWordsRemoved = [token.decode("windows-1252") for token in tokens if token not in stopWords]
    stems = [porter.stem(word) for word in stopWordsRemoved]
    st = ' '
    result = st.join(stems)
    result = [result]
    print (result)
    spars=count_vect.transform(result)
    tfidf=tfidf_transformer.transform(spars)
    print (tfidf.shape)
    
    predicted=clf.predict(tfidf)
    predict_ranking = clf.predict_proba(tfidf)
    print ("choosen class " ,predicted.item())
    print ("ranking probability ", predict_ranking)
    return predicted , clf , tfidf

      
    

train()
my_str = input("Please inter desired sentence")
print (my_str)

testSentence = my_str
predict , clf ,tfidf  = evaluate (testSentence)
Python
<pre lang="Python">
Posted
Updated 14-Feb-17 4:36am
Comments
Richard MacCutchan 14-Feb-17 12:48pm    
Where does the error occur?
Akbar Fardi 14-Feb-17 14:40pm    
when i run the code .this error occur in console
Richard MacCutchan 15-Feb-17 3:09am    
Where in the code does it occur?

This content, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)



CodeProject, 20 Bay Street, 11th Floor Toronto, Ontario, Canada M5J 2N8 +1 (416) 849-8900