Charting AI/ML Growth Across the World

import warnings
warnings.filterwarnings('ignore')

import os
import random
import re
import string
import tabulate
from collections import Counter

import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

from matplotlib import rc
rc('animation', html='jshtml')

import seaborn as sns

import re, os, string
from sklearn.feature_extraction.text import TfidfVectorizer

from gensim.models.doc2vec import Doc2Vec, TaggedDocument
from gensim.models import Word2Vec

from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import nltk
nltk.download('punkt')
nltk.download("stopwords")

from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import MiniBatchKMeans
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn.decomposition import PCA

import networkx as nx

import plotly.express as px
import plotly.graph_objects as go
import plotly.io as pio
SEED = 42
random.seed(SEED)
os.environ["PYTHONHASHSEED"] = str(SEED)
np.random.seed(SEED)

# get the list of the top 10 universities till now
dict_ = {}
for insti in papers_df['Affiliation']:
    insti = str(insti)
    if insti == 'None':
        continue
    if insti not in dict_.keys():
        dict_[insti] = (papers_df.Affiliation == insti).sum()

sorted_by_value=(sorted(dict_.items(), key=lambda item: item[1], reverse=True))
top_10_institutes = []
for tuple_ in sorted_by_value[:10]:
    top_10_institutes.append(tuple_[0])

# get a dictionary which outputs the list of number of papers published by top 10 universities in a year x:
y_paper_count_top_10 = {}

for year in range(2006,2022, 1):
    y_paper_count_top_10[year] = []
    temp_df = papers_df[papers_df.Year == year]
    for insti in top_10_institutes:
        count = (temp_df.Affiliation==insti).sum()
        y_paper_count_top_10[year].append(count)

# Trend for author count every year
# First we get author list for one year, then we see if it was in the past year or not.

# Assumption is that once we seen an author has published in some year, then that author remains active subsequent years.
# This trend is increasing (either constant or increases)

# List of year in dataset:
year_list = sorted(list(set(papers_df["Year"])))

dict_year_author = {}
dict_year_author_cumu = {}

for year in year_list:
    dict_year_author[str(year)] = list(set(papers_df.loc[papers_df["Year"]==year]["Author"]))

dict_year_author_cumu[str(year_list[0])] = dict_year_author[str(year_list[0])]

for i in range(1,len(year_list)):
    key_ = str(year_list[i])
    prev_key = str(year_list[i-1])
    dict_year_author_cumu[str(key_)] = list(set(dict_year_author[key_] + dict_year_author_cumu[prev_key]))

# Now we try to answer the question in top 5% author, how many of them collab with each other
top_author_collab = pd.read_csv('data/new_papers.csv')
top_author_collab = top_author_collab.drop_duplicates()

author = []
value = []
# for name, pub in top_author_collab.Author.value_counts().head(len(top_author_collab)//100).iteritems():
for name, pub in top_author_collab.Author.value_counts().head(50).iteritems():
    author.append(name)
    value.append(pub)

top_author_collab = top_author_collab.loc[top_author_collab["Author"].isin(author)].copy()

new_df = top_author_collab[['Conference','Year','Title', 'Author']].groupby(['Conference','Year','Title'])

Authors_collab = []
key_seen = []

for i in top_author_collab.index:
    curr_key = tuple(top_author_collab.loc[i,['Conference','Year','Title']])
    if curr_key not in key_seen:
        key_seen.append(curr_key)
        lst_author = list(new_df.get_group(curr_key).Author)
        lst_author.sort()
        if lst_author not in Authors_collab:
            Authors_collab.append(lst_author)

# Author collab stores all the edges
node_list = list(set(top_author_collab.Author))
edge_list = [i for i in Authors_collab if len(i)>=2]

# creating the graph

import networkx as nx
import matplotlib.pyplot as plt
import itertools
from pyvis.network import Network
# creating notebook
net = Network(notebook = False)

class GraphVisualization:
       
    def __init__(self):
        G = None
        self.visual = []
          
    # addEdge function inputs the vertices of an
    # edge and appends it to the visual list
    def addEdges(self, list_edges):
        to_add = list(itertools.permutations(list_edges, 2))
        for i in to_add:
            self.visual.append(list(i))
          
    # In visualize function G is an object of
    # class Graph given by networkx G.add_edges_from(visual)
    # creates a graph with a given list
    # nx.draw_networkx(G) - plots the graph
    # plt.show() - displays the graph
    def visualize(self):
        self.G = nx.Graph()
        self.G.add_edges_from(self.visual)
        # nx.draw_spring(G)
        pos = nx.random_layout(self.G)
        nx.draw(self.G, pos , with_labels = True, width=0.4, 
            node_color='lightblue', node_size=200)
        plt.figure(figsize=(13,13), dpi=300)
    
    def getGraphObj(self):
        self.G = nx.Graph()
        self.G.add_edges_from(self.visual)
        return self.G

Graph = GraphVisualization()
for i in edge_list:
    Graph.addEdges(i)

## VISUALISING USING PYVIS
nx_graph = Graph.getGraphObj()


net.from_nx(nx_graph, show_edge_weights=False)
net.width = '1000px'
net.show("../_html/graph.html")

PUNCTUATION = """!"#$%&'()*+,-./:;<=>?@[\]^_ *{|}~""" 
TOP_K_KEYWORDS = 10 # top k number of keywords to retrieve in a ranked document
STOPWORD_PATH = 'data/stopwords_ls.txt'
PAPERS_PATH = 'data/papers.csv'

def get_sw_lst(path):
    
    with open(path,'r') as f:

        listl=[]
        for line in f:
            strip_lines=line.strip()
            listli=strip_lines.split()
            listl += listli
            
        return listl

def clean_text(text):
    """Doc cleaning"""
    
    # Lowering text
    text = text.lower()
    
    # Removing punctuation
    text = "".join([c for c in text if c not in PUNCTUATION])
    
    # Removing whitespace and newlines
    text = re.sub('\s+',' ',text)
    
    return text

def sort_coo(coo_matrix):
    """Sort a dict with highest score"""
    tuples = zip(coo_matrix.col, coo_matrix.data)
    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)

def extract_topn_from_vector(feature_names, sorted_items, topn=10):
    """get the feature names and tf-idf score of top n items"""
    
    #use only topn items from vector
    sorted_items = sorted_items[:topn]

    score_vals = []
    feature_vals = []
    
    # word index and corresponding tf-idf score
    for idx, score in sorted_items:
        
        #keep track of feature name and its corresponding score
        score_vals.append(round(score, 3))
        feature_vals.append(feature_names[idx])

    #create a tuples of feature, score
    results= {}
    for idx in range(len(feature_vals)):
        results[feature_vals[idx]]=score_vals[idx]
    
    return results

def get_keywords(vectorizer, feature_names, doc):
    """Return top k keywords from a doc using TF-IDF method"""

    #generate tf-idf for the given document
    tf_idf_vector = vectorizer.transform([doc])
    
    #sort the tf-idf vectors by descending order of scores
    sorted_items=sort_coo(tf_idf_vector.tocoo())

    #extract only TOP_K_KEYWORDS
    keywords=extract_topn_from_vector(feature_names,sorted_items,TOP_K_KEYWORDS)
    
    return list(keywords.keys())

test = data.groupby('year').sample(n= 25, random_state=25)
train = data.drop(test.index, axis = 0)

corpora = train['full_text'].to_list()

# get the stop words list
stopwords=get_sw_lst(STOPWORD_PATH)

# Initializing TF-IDF Vectorizer with the given stopword list
vectorizer = TfidfVectorizer(stop_words=stopwords, smooth_idf=True, use_idf=True)

# Creating vocabulary with our corpora
vectorizer.fit_transform(corpora)

# Storing the created vocabulary
feature_names = vectorizer.get_feature_names()

result = []
i = 0
test_lst = test['full_text'].to_list()
yrs = test.year.to_list()

word_data = {}

# Associating the top 10 keywords from each paper and their years
for doc in test_lst:
    df = {}
    df['full_text'] = doc
    df['year'] = yrs[i]
    wordsl = get_keywords(vectorizer, feature_names, doc)
    df['top_keywords'] = wordsl
    result.append(df)

    if(yrs[i] in word_data):
        word_data[yrs[i]] += wordsl
    else:
        word_data[yrs[i]] = wordsl

    i = i+1
    
final = pd.DataFrame(result)

# importing all necessary modules
from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt

 
for key,val in word_data.items():
    words_string=(" ").join(val)
    wordcloud = WordCloud(width = 1000, height = 500).generate(words_string)
    plt.figure(figsize=(15,8))
    plt.imshow(wordcloud)
    # plt.title('Word Cloud for the year {}'.format(str(i)))
    plt.axis("off")
    plt.savefig("assets/wc_{}.png".format(key), bbox_inches='tight')
    # plt.show()
    plt.close()

import imageio
import matplotlib.image as mpimg
import matplotlib.pyplot as plt

for i in range(1987,2020):

    x = 'assets/wc_{}.png'.format(i)
    img = mpimg.imread(x) 

    plt.figure(figsize=(15,8), facecolor='black')
    # plt.imshow(img)
    plt.axis("off")
    plt.title('Word Cloud for the year {}'.format(str(i)), color = 'white')
    plt.imshow(img)
    plt.savefig("assets/NTwc_{}.png".format(i), bbox_inches='tight')

images = {}
for year in range(1987, 2020):
    x = './assets/NTwc_{}.png'.format(year)
    images[x] = str(year)

new_images = []
for filename in list(images.keys()):
    new_images.append(imageio.imread(filename))

imageio.mimsave('./assets/gif/movie.gif', new_images, duration = 2)