# Load Table of Contentssoup = bs4.BeautifulSoup(open('./ECS MA2019-02.003/Abstracts/tos.html', encoding='utf-8'),'lxml')# Function to remove html tagspx = re.compile(r"<[^>]*?>")#Load Categoriesp_len = len(soup('h2'))categorys = []for i in range(p_len):    c = px.sub("", str(soup('h2')[i]))    categorys.append(c)print("categoly_num ="+str(p_len))print(categorys)

#Load Sessionsp_len = len(soup('h3'))sessions = []for i in range(p_len):    c = px.sub("", str(soup('h3')[i]))    sessions.append(c)print("session_num ="+str(p_len))print(sessions)

#Load stopwordswith open('stopwords.txt', 'r', encoding='utf-8') as f:    stops = f.readlines()stopwords = []for s in stops:    s = s.replace("\n", "")    stopwords.append(s)

dic_all = []# make session number and name dictionarydic_sessions = {}for s in sessions:    id_ = s.split('-',1)    dic_sessions[id_[0]]=id_[1]#Get all pathes of abstract .html filepath = os.listdir("./ECS MA2019-02.003/Abstracts/Html")#Scraping html documentsfor n,i in enumerate(tqdm(path)):    soup = bs4.BeautifulSoup(open("./ECS MA2019-02.003/Abstracts/Html/"+i, encoding='utf-8'), "html5lib")    #Get title, authors, affiliation    title =px.sub("", soup.title.string)    author = px.sub("", str(soup('p')[1]))    author = author.split(', ')    affiliation = px.sub("", str(soup('p')[2]))    affiliation = affiliation.replace("a", "", 1)    #Get presentation number, session    number = re.search('-[0-9]{4}', str(title))    number = number.group().replace("-", "")    session = re.match('.*[0-9]{2}', str(title))    session = session.group()    #Remove session name in title    title = re.sub('\w[0-9]{2}-[0-9]{4}', "", title)    title = title.replace(u"\xa0",u"")    title = title.replace(session, '')    title = title.replace('(Invited)', '')    session = session.replace(number, '')    session = session.replace('-', '')    try:        session_name = dic_sessions[session]    except:        session = session.split(u'\xa0',1)[0]        session_name = dic_sessions[session]    session_name = session_name.replace(u'&', '&')    #Get main text and reshape it    content = px.sub("", str(soup.find(class_=re.compile('contaner_content'))))    stoplist = [",", ".", ":", ";", "(", ")", "{", "}", "[", "]", "\'", "\""]    for s in stoplist:        content = content.replace(s, "")    content = content.replace("\t", "")    #Get word list for Word2Vec Learning    c_list = content.split(" ")    c_list = [c.lower() for c in c_list]    for s in stopwords:        c_list = [c for c in c_list if i != s]    c_list = [c for c in c_list if c != ""]    #Summarize it as dictionary    dic ={'num':number,'title':title,'author':author,          'affiliation':affiliation,'session':session, 'session_name':session_name,          'contents':content, 'mod_contents': c_list, 'vector':n, 'url':i}    dic_all.append(dic)

{'num': '0001', 'title': 'The Impact of Coal Mineral Matter (alumina and silica) on Carbon Electrooxidation in the Direct Carbon Fuel Cell', 'author': ['Simin Moradmanda', 'Jessica A Allena', 'Scott W Donnea'], 'affiliation': 'University of Newcastle', 'session': 'A01', 'session_name': 'Battery and Energy Technology Joint General Session', 'contents': 'Direct carbon fuel cell DCFC as an electrochemical device...', 'mod_contents': ['direct','carbon','fuel','cell', ... ,'melting'], 'vector': 0, 'url': '1.html'}

charactors = [i[0] for i in categorys]cat_counts = np.zeros(len(charactors))authors_counts = {}words_counts = {}for c in charactors:    authors_counts[c] = []    words_counts[c] = []for d in dic_all:    char = d['session'][0]    cat_counts[charactors.index(char)] += 1    authors_counts[char].append(len(d['author']))    words_counts[char].append(len(d['contents'].split(' ')))

fig = plt.figure(figsize=(10,4),dpi=200)plt.bar(charactors, cat_counts, label=categorys)plt.xlabel('Categories',fontsize=14)plt.title('Number of presentation',fontsize=16)plt.show()

points = tuple(authors_counts[i] for i in charactors)fig = plt.figure(figsize=(10,4),dpi=200)plt.boxplot(points)plt.gca().set_xticklabels(charactors)plt.xlabel('Categories',fontsize=14)plt.title('Number of author',fontsize=16)plt.show()

import collectionsaffiliation_list = []for a in dic_all:    affiliation_list.extend(a['affiliation'])   a_count = collections.Counter(affiliation_list)a_sorted = sorted(a_count.items(), key=lambda x:x[1], reverse=True)