added Lina primary & return image from duck duck go

Conversations/Lina primary.csv

@@ -0,0 +1,65 @@
+what is your name,i am lina
+lina,"yes , how can i help you"
+are you lina,yes this would be me
+how old are you,"i am 23 , still young !! :D"
+who are you,i am your friendly assistant
+who made you,i was developed by a team of expetianaly inteligant engineers
+who is youssef,he helped me with my heart
+who is anwar,he helped me with my heart
+who is anwar,my father
+who is youssef,my father
+who is ezzat,he developed me
+who is karim,he helped me talk
+who is ezzat,he would be my father
+who is karim,you are taking about my father
+who is zaki,ooh ... he would be my love
+how were you built,a smart combination of TF-IDF & pattern recognition
+what is machine learning,this is the way I learn
+what is NLP,this is how I understand you
+where are you from,my hometown is Cairo
+what does TF-IDF stand for,Term freq - inverse document freq
+what is TF-IDF,this is how I come to understand how similar sentences are 
+how were you integrated,"developed on a web server , thanks to karim , processed by anwar , youssef , zaki , accssed by ezzat , it was quite fun learing from them"
+how can you help me ,"I am your friend , I help you with your work , and you can talk to me whenever you feel bored"
+how can you help me ,"I can manage your meeting , help you with your emails , I can reaaly help you a lot"
+can you help me,this is why I am here
+who are you,"I am Lina , your friend "
+how old are you ?,I would rather not say 
+would you like to help,I like helping others
+what are you made of ?,a combination of TF-IDF & CKY & flesh & Blood
+what can you do,"I can help you with your message , I can help you with your work , I can answer your questions , try asking me anything"
+can you answer my questions,"try me out , I can answer quite anything"
+ok,glad to hear this
+what ?,I can't understand
+why ?,what do you mean ??
+where ?,where what ??
+when ?,"when !!??  , what do mean ??"
+yes,I always knew that :D
+no,sorry for this 
+no,ok no .. Don't be mad
+yes,gald to hear this
+me too,me three .. I mean .. Me also :D
+glad,I am also glad :D
+bitch,why would you say so :( 
+fuck,this is not nice :( 
+you too,do you think so ?? 
+you ,me ??!!
+i,you what .. I can't figure out ?
+how was the work divided among you ?,"Youssef , anwar , zaki worked on my heart .. I mean � worked on my core functionality ..karim worked on hosting the web service online � and ezzat called it from his mobile app .. It was fun working with them .. Really"
+how were you built,"Youssef , anwar , zaki worked on my heart .. I mean � worked on my core functionality ..karim worked on hosting the web service online � and ezzat called it from his mobile app .. It was fun woring with them .. Really"
+how were you developed,"Youssef , anwar , zaki worked on my heart .. I mean � worked on my core functionality ..karim worked on hosting the web service online � and ezzat called it from his mobile app .. It was fun woring with them .. Really"
+what do you know about AinShams ?,this is my home
+what is AinShams,you are talking about my Home
+who is Dr Islam ?,Dr Islam helped me become who I am .. I would always be in debt
+would you think we will succeed ?,Bad3elko � 2ool ya Rab 
+will we succeed ?,Bad3elko � 2ool ya Rab 
+what degree should they have ,I would please you for emtyaaz � they earn it walahy 
+what do they earn,I think they earn emtyaazz � they worked hard to develop me .. I swear 
+thank you,I couldn't be more happy
+thank you,I am just happy I have helped you
+fuck you lina,I won't talk to you �. That�s ain't nice 
+whats your best characher,I like being casual
+who is your boyfriend,that would be my zaki
+who is your love ?,zaki all the way
+who do you love the most,that would be my beloved developers 
+are you in love ,that place is taken for zaki

Lina.py

@@ -139,14 +139,19 @@ def parse(sentence):
         print("BeautifulSoup succeeded")
 
         answer = soup_super_page.findAll('Abstract')[0].text
+        Image =  soup_super_page.findAll('Image')[0].text
         if (answer == ""):
             answer = soup_super_page.findAll('Text')[0].text
-        return True, answer
+
+        return True, answer , Image
     except Exception as exception:
         print ("error2", exception)
         print (type(exception).__name__)
         print (exception.__class__.__name__)
-        return False, ""  # -----------------------General DataSet   &   Movies Lines----------------#
+        return False, "" 
+
+
+# -----------------------General DataSet   &   Movies Lines----------------#
 
 
 def talk_to_lina(test_set_sentence, csv_file_path, tfidf_vectorizer_pikle_path, tfidf_matrix_train_pikle_path):
@@ -239,6 +244,96 @@ def talk_to_lina(test_set_sentence, csv_file_path, tfidf_vectorizer_pikle_path,
                 break
 
 
+def talk_to_lina_primary(test_set_sentence, csv_file_path, tfidf_vectorizer_pikle_path, tfidf_matrix_train_pikle_path):
+    i = 0
+    sentences = []
+
+    # enter your test sentence
+    test_set = (test_set_sentence, "")
+
+    # 3ashan yzabt el indexes
+    sentences.append(" No you.")
+    sentences.append(" No you.")
+
+    try:
+        ##--------------to use------------------#
+        f = open(tfidf_vectorizer_pikle_path, 'rb')
+        tfidf_vectorizer = pickle.load(f)
+        f.close()
+
+        f = open(tfidf_matrix_train_pikle_path, 'rb')
+        tfidf_matrix_train = pickle.load(f)
+        f.close()
+        # ----------------------------------------#
+    except:
+        # ---------------to train------------------#
+        start = timeit.default_timer()
+
+        # enter jabberwakky sentence
+        with open(csv_file_path, "r") as sentences_file:
+            reader = csv.reader(sentences_file, delimiter=',')
+            # reader.next()
+            # reader.next()
+            for row in reader:
+                # if i==stop_at_sentence:
+                #    break
+                sentences.append(row[0])
+                i += 1
+
+        tfidf_vectorizer = TfidfVectorizer()
+        tfidf_matrix_train = tfidf_vectorizer.fit_transform(sentences)  # finds the tfidf score with normalization
+        # tfidf_matrix_test =tfidf_vectorizer.transform(test_set)
+        stop = timeit.default_timer()
+        print ("training time took was : ")
+        print stop - start
+
+        f = open(tfidf_vectorizer_pikle_path, 'wb')
+        pickle.dump(tfidf_vectorizer, f)
+        f.close()
+
+        f = open(tfidf_matrix_train_pikle_path, 'wb')
+        pickle.dump(tfidf_matrix_train, f)
+        f.close()
+        # -----------------------------------------#
+
+    tfidf_matrix_test = tfidf_vectorizer.transform(test_set)
+
+    cosine = cosine_similarity(tfidf_matrix_test, tfidf_matrix_train)
+
+    cosine = np.delete(cosine, 0)
+    max = cosine.max()
+    response_index = 0
+    if (max > 0.9):
+        new_max = max - 0.01
+        list = np.where(cosine > new_max)
+        print ("number of responses with 0.01 from max = " + str(list[0].size))
+        response_index = random.choice(list[0])
+
+    else:
+        print ("not sure")
+        print ("max is = " + str(max))
+        response_index = np.where(cosine == max)[0][0] + 2  # no offset at all +3
+        return "null", "null",
+
+    j = 0
+
+    with open(csv_file_path, "r") as sentences_file:
+        reader = csv.reader(sentences_file, delimiter=',')
+        for row in reader:
+            j += 1  # we begin with 1 not 0 &    j is initialized by 0
+            if j == response_index:
+
+                if delimeter in row[1]:
+                    # get newest suggestion
+                    answer_row = row[1].split(delimeter)
+                    row[1] = answer_row[1]
+
+                else:  # add new suggestion
+                    note = "just return old original suggestion"
+
+                return row[1], response_index,
+                break
+
 # -------------------------------------------------------------------------#
 
 # -----------------------Edit Module (RealTime Learn)----------------------#
@@ -295,6 +390,19 @@ def edit_real_time(new_sentence, dataset_number, LineID):
 
 
 def callBot(var, option):
+    "Lina primary.csv"
+    get_relative_path("action_conversation.csv")
+
+    Lina_all_path_primary                 =    get_relative_path("Lina primary.csv")
+    tfidf_vectorizer_april_path_primary   =    get_relative_path("tfidf_vectorizer_april_primary.pickle")
+    tfidf_matrix_train_april_path_primary =    get_relative_path("tfidf_matrix_train_april_primary.pickle")
+
+    response_primary , line_id_primary  = talk_to_lina_primary(var, Lina_all_path_primary, tfidf_vectorizer_april_path_primary,
+                                                 tfidf_matrix_train_april_path_primary)
+
+    if (response_primary !="null") :
+            return "message", (response_primary.capitalize().strip(), option, line_id_primary)
+
     result = extract_intents(var)
 
     response = ""
@@ -304,7 +412,7 @@ def callBot(var, option):
         if (fact_question[0]):
             print "Fact Question"
             # print fact_question[1].encode('utf-8')
-            response = fact_question[1].encode('utf-8').split('.')[0] + '.'
+            response = fact_question[1].encode('utf-8').split('.')[0] + '.'   +    fact_question[2]
             print
 
         else:
@@ -387,3 +495,9 @@ def get_relative_path(filename):
     conversations_dir = os.path.join(dir, "Conversations")
     relative_path = os.path.join(conversations_dir, filename)
     return relative_path
+
+
+#local test 
+while 1:
+    var = raw_input("Talk to Lina: ")
+    print  callBot(var , 1)