I have developed a web scrapping code in Python which takes data from Hattrick.org's matches and returns them in a table so it can be mined, determined likelihood of goals, etc.
I have the difficult that is really slow, returning 12.000 rows in 5 hours or so.
This question is to ask if there is a way to improve the web scrapping technique so it does not take that amount of time.
This is the code in Python.
import requests
from bs4 import BeautifulSoup
import re
import pandas as pd
import numpy as np
ini = 631163587
q = 200000 # Change to q = 10 to try a sample
Cols = {'01. Local MF',
'02. Away MF',
'03. Local RD',
'04. Away RD',
'05. Local CD',
'06. Away CD',
'07. Local LD',
'08. Away LD',
'09. Local RA',
'10. Away RA',
'11. Local CA',
'12. Away CA',
'13. Local LA',
'14. Away LA',
'15. Local IndD',
'16. Away IndD',
'17. Local IndA',
'18. Away IndA',
'19. Local Attitude',
'20. Away Attitude',
'21. Local Tactic',
'22. Away Tactic',
'23. Local Tactic Level',
'24. Away Tactic Level',
'25. Local Score',
'26. Away Score'}
df_ht = pd.DataFrame(data=np.nan,index=range(ini,ini+q),columns=Cols)
cont=[]
for i in range(ini,ini+q):
url2 = 'https://www74.hattrick.org/Club/Matches/Match.aspx?matchID='+str(i)
response = requests.get(url2)
soup = BeautifulSoup(response.text, 'html.parser')
s1 = soup.findAll('td')
m = soup.findAll('meta')[10].attrs['content']
d = re.findall('[ ,.,A-Z,a-z,0-9]* - [., ,A-Z,a-z,0-9]*',m)
d2 = re.findall('[0-9]+',d[1])
partido = d[0]
try:
D = {'01. Local MF': float(s1[3].contents[0]),
'02. Away MF': float(s1[4].contents[0]),
'03. Local RD': float(s1[10].contents[0]),
'04. Away RD': float(s1[11].contents[0]),
'05. Local CD': float(s1[17].contents[0]),
'06. Away CD': float(s1[18].contents[0]),
'07. Local LD': float(s1[24].contents[0]),
'08. Away LD': float(s1[25].contents[0]),
'09. Local RA': float(s1[31].contents[0]),
'10. Away RA': float(s1[32].contents[0]),
'11. Local CA': float(s1[38].contents[0]),
'12. Away CA': float(s1[39].contents[0]),
'13. Local LA': float(s1[45].contents[0]),
'14. Away LA': float(s1[46].contents[0]),
'15. Local IndD': float(s1[54].contents[0]),
'16. Away IndD': float(s1[55].contents[0]),
'17. Local IndA': float(s1[61].contents[0]),
'18. Away IndA': float(s1[62].contents[0]),
'19. Local Attitude': (s1[67].contents[0]),
'20. Away Attitude': (s1[68].contents[0]),
'21. Local Tactic': s1[70].contents[0],
'22. Away Tactic': s1[71].contents[0],
'23. Local Tactic Level': s1[75].contents[0],
'24. Away Tactic Level': s1[76].contents[0],
'25. Local Score': float(d2[0]),
'26. Away Score': float(d2[1])}
df_ht.loc[i,:] = D
except:
cont.append(i)
df_ht.to_csv(r"Datos9.csv")
