Re-organize directory structure (#4971)

See discussion in #2491

Co-authored-by: Ivan Levkivskyi <ilevkivskyi@dropbox.com>
This commit is contained in:
Ivan Levkivskyi
2021-01-27 12:00:39 +00:00
committed by GitHub
parent 869238e587
commit 16ae4c6120
1399 changed files with 601 additions and 97 deletions

View File

@@ -0,0 +1,19 @@
import sys
from typing import Iterable, List, NamedTuple, Optional
class _RequestRate(NamedTuple):
requests: int
seconds: int
class RobotFileParser:
def __init__(self, url: str = ...) -> None: ...
def set_url(self, url: str) -> None: ...
def read(self) -> None: ...
def parse(self, lines: Iterable[str]) -> None: ...
def can_fetch(self, user_agent: str, url: str) -> bool: ...
def mtime(self) -> int: ...
def modified(self) -> None: ...
def crawl_delay(self, useragent: str) -> Optional[str]: ...
def request_rate(self, useragent: str) -> Optional[_RequestRate]: ...
if sys.version_info >= (3, 8):
def site_maps(self) -> Optional[List[str]]: ...