In [68]: #TASK 1

#import all required libraray import pandas as pd import math import numpy
as np from scipy import sparse from scipy.stats import uniform from
sklearn.feature_extraction.text import TfidfVectorizer

# input data string corpus = ['this is the first document',

'this document is the second document', 'and this is the third one', 'is
this the first document'] # use fit method to compute Bag of words

vectorizer = TfidfVectorizer() vectorizer.fit(corpus) skl_output =

vectorizer.transform(corpus) bow=vectorizer.get_feature_names() print(bow)
IDF_reference=vectorizer.idf_ print(IDF_reference) #compute IDF using
custom method

from sklearn.feature_extraction.text import CountVectorizer

vectorizer = CountVectorizer() vectors = vectorizer.fit_transform(corpus)

matrix = CountVectorizer() matrix.fit(corpus) # after this statement the

matrix will build the vocabulary with all th e unique words

Create PDF in your applications with the Pdfcrowd HTML to PDF API PDFCROWD
# you should call this function only after fit()

# to convert the sentance into numerical vectors, we will call transfor

m() # the first feature name will corresponds to first column in
transforme d matrix # the 2nd feature name will corresponds to 2nd column
in transformed ma trix

print(matrix.transform(corpus).toarray())

# Here we will print the sklearn tfidf vectorizer idf values after appl
ying the fit method # After using the fit function on the corpus the vocab
has 9 words in i t, and each has its idf value.

#compute IDF using custom method

for i in range(len(bow)):
Y=0 word=bow[i] for j in range(len(corpus)):
list[j]=corpus[j].split()

if(word in list[j]): #print(word) #print(list[j]) Y=Y+1 X=len(corpus)

XY=math.log((1+X)/(1+Y)) IDF_custom=XY+1 print(IDF_custom)

['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'th

is'] [1 91629073 1 22314355 1 51082562 1 1 91629073 1 91629073 Create PDF

in your applications with the Pdfcrowd HTML to PDF API PDFCROWD

[1.91629073 1.22314355 1.51082562 1. 1.91629073 1.91629073
1. 1.91629073 1. ] [[0 1 1 1 0 0 1 0 1] [0 2 0 1 0 1 1 0 1] [1 0 0 1 1 0 1
1 1] [0 1 1 1 0 0 1 0 1]] 1.916290731874155 1.2231435513142097
1.5108256237659907 1.0 1.916290731874155 1.916290731874155 1.0
1.916290731874155 1.0

In [15]: #TASK2
import pickle import numpy as np with
open("E:\Applied_AI\Assignments\cleaned_strings","rb") as f:
data = pickle.load(f) # printing the length of the corpus loaded
print("Number of documents in data = ",len(data))

#call all usique words using fit and tranform function from
sklearn.feature_extraction.text import TfidfVectorizer vectorizer =
TfidfVectorizer() vectorizer.fit(data) skl_output =
vectorizer.transform(data) bow=vectorizer.get_feature_names()

#compute IDF IDF=vectorizer.idf_

#sort IDF in descending order sorted_IDF=np.sort(IDF)

required_IDF=sorted_IDF[::-1]

#print top 50 IDF values

Create PDF in your applications with the Pdfcrowd HTML to PDF API PDFCROWD
print(required_IDF[0:49])

Number of documents in data = 746 [6.922918 6.922918 6.922918 6.922918

6.922918 6.922918 6.922918 6.92291 8 6.922918 6.922918 6.922918 6.922918
6.922918 6.922918 6.922918 6.92291 8 6.922918 6.922918 6.922918 6.922918

6.922918 6.922918 6.922918 6.92291 8 6.922918 6.922918 6.922918 6.922918

6.922918 6.922918 6.922918 6.92291 8 6.922918 6.922918 6.922918 6.922918

6.922918 6.922918 6.922918 6.92291 8 6.922918 6.922918 6.922918 6.922918

6.922918 6.922918 6.922918 6.92291 8 6.922918]

Create PDF in your applications with the Pdfcrowd HTML to PDF API PDFCROWD