English to French Translator

Author: pawankg

English_to_French_Language_translator/III_Machine_Translation.ipynb

@@ -0,0 +1,77 @@
+import re
+import string
+
+import numpy as np
+import pandas as pd
+from nltk.corpus import stopwords
+from nltk.stem import PorterStemmer
+from nltk.tokenize import TweetTokenizer
+
+
+def process_tweet(tweet):
+    '''
+    Input:
+        tweet: a string containing a tweet
+    Output:
+        tweets_clean: a list of words containing the processed tweet
+
+    '''
+    stemmer = PorterStemmer()
+    stopwords_english = stopwords.words('english')
+    # remove stock market tickers like $GE
+    tweet = re.sub(r'\$\w*', '', tweet)
+    # remove old style retweet text "RT"
+    tweet = re.sub(r'^RT[\s]+', '', tweet)
+    # remove hyperlinks
+    tweet = re.sub(r'https?:\/\/.*[\r\n]*', '', tweet)
+    # remove hashtags
+    # only removing the hash # sign from the word
+    tweet = re.sub(r'#', '', tweet)
+    # tokenize tweets
+    tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True,
+                               reduce_len=True)
+    tweet_tokens = tokenizer.tokenize(tweet)
+
+    tweets_clean = []
+    for word in tweet_tokens:
+        if (word not in stopwords_english and  # remove stopwords
+            word not in string.punctuation):  # remove punctuation
+            # tweets_clean.append(word)
+            stem_word = stemmer.stem(word)  # stemming word
+            tweets_clean.append(stem_word)
+
+    return tweets_clean
+
+
+def get_dict(file_name):
+    """
+    This function returns the english to french dictionary given a file where the each column corresponds to a word.
+    Check out the files this function takes in your workspace.
+    """
+    my_file = pd.read_csv(file_name, delimiter=' ')
+    etof = {}  # the english to french dictionary to be returned
+    for i in range(len(my_file)):
+        # indexing into the rows.
+        en = my_file.loc[i][0]
+        fr = my_file.loc[i][1]
+        etof[en] = fr
+
+    return etof
+
+
+def cosine_similarity(A, B):
+    '''
+    Input:
+        A: a numpy array which corresponds to a word vector
+        B: A numpy array which corresponds to a word vector
+    Output:
+        cos: numerical number representing the cosine similarity between A and B.
+    '''
+    # you have to set this variable to the true label.
+    cos = -10
+    dot = np.dot(A, B)
+    norma = np.linalg.norm(A)
+    normb = np.linalg.norm(B)
+    cos = dot / (norma * normb)
+
+    return cos