We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 360214e commit fddd858Copy full SHA for fddd858
Doc/library/urllib.robotparser.rst
@@ -91,16 +91,16 @@ class::
91
92
>>> import urllib.robotparser
93
>>> rp = urllib.robotparser.RobotFileParser()
94
- >>> rp.set_url("http://www.musi-cal.com/robots.txt")
+ >>> rp.set_url("http://www.pythontest.net/robots.txt")
95
>>> rp.read()
96
>>> rrate = rp.request_rate("*")
97
>>> rrate.requests
98
- 3
+ 1
99
>>> rrate.seconds
100
- 20
101
>>> rp.crawl_delay("*")
102
6
103
- >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco")
104
- False
105
- >>> rp.can_fetch("*", "http://www.musi-cal.com/")
+ >>> rp.can_fetch("*", "http://www.pythontest.net/")
106
True
+ >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/")
+ False
0 commit comments