ulfgebhardt/college
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

last small fix for today

Browse Source
This commit is contained in:
Michael Scholz 2013-05-14 18:47:34 +02:00
parent 2e6037954b
commit a136dc18f5
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ss2013/1_Web Mining/Uebungen/2_Uebung/crawler/crawler.py
View File

@ -50,8 +50,6 @@ def checkRobotsTxt(url):
else:
if rp.can_fetch("*", url):
print "checking robots.txt ("+robotsUrl+") \n "+colored("-> allowed to visit :) "+url, "green")
global visitedSites
visitedSites += 1
return True
else:
print "checking robots.txt ("+robotsUrl+") \n "+colored("-> not allowed to visit :( "+url, "red")
@ -139,6 +137,8 @@ while(i <= numberOfSites):
try:
page = urllib.urlopen(url)
extractor.feed(page.read())
global visitedSites
visitedSites += 1
except:
print colored("("+str(i)+"/"+str(numberOfSites)+") can't read url: "+url, "red")
i += 1