🌐 Python Requests Module - Complete Guide#

1
import requests

1
import requests
2
print(requests.__version__)

1
import requests
2

3
# Make a GET request
4
response = requests.get("https://httpbin.org/get")
5

6
# Print response
7
print(response.text)

1
response = requests.get("https://httpbin.org/get")
2

3
print(response.status_code)     # 200
4
print(response.ok)              # True (if status < 400)
5
print(response.reason)          # OK

1
import requests
2

3
# POST with form data
4
data = {
5
    "username": "admin",
6
    "password": "secret123"
7
}
8

9
response = requests.post("https://httpbin.org/post", data=data)
10
print(response.text)

1
import requests
2

3
# POST with JSON body
4
json_data = {
5
    "name": "Ahmed",
6
    "age": 25,
7
    "city": "Casablanca"
8
}
9

10
response = requests.post("https://httpbin.org/post", json=json_data)
11
print(response.json())

1
import requests
2

3
headers = {
4
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
5
    "Accept": "application/json",
6
    "Authorization": "Bearer your_token_here",
7
    "X-Custom-Header": "custom_value"
8
}
9

10
response = requests.get("https://httpbin.org/headers", headers=headers)
11
print(response.json())

1
response = requests.get("https://httpbin.org/get")
2

3
# All headers
4
print(response.headers)
5

6
# Specific header
7
print(response.headers["Content-Type"])
8
print(response.headers.get("Server"))

1
response = requests.get("https://httpbin.org/get?name=Ahmed&age=25")

1
import requests
2

3
params = {
4
    "name": "Ahmed",
5
    "age": 25,
6
    "city": "Casablanca"
7
}
8

9
response = requests.get("https://httpbin.org/get", params=params)
10
print(response.url)
11
# Output: https://httpbin.org/get?name=Ahmed&age=25&city=Casablanca

1
params = {
2
    "tags": ["python", "hacking", "security"]
3
}
4

5
response = requests.get("https://httpbin.org/get", params=params)
6
print(response.url)
7
# Output: https://httpbin.org/get?tags=python&tags=hacking&tags=security

1
import requests
2

3
response = requests.get("https://httpbin.org/get")
4

5
# Status
6
print(response.status_code)      # 200
7
print(response.ok)               # True
8
print(response.reason)           # OK
9

10
# Content
11
print(response.text)             # Response as string
12
print(response.content)          # Response as bytes
13
print(response.json())           # Response as dict (if JSON)
14

15
# Headers
16
print(response.headers)          # Response headers
17
print(response.request.headers)  # Request headers
18

19
# URL
20
print(response.url)              # Final URL (after redirects)
21

22
# Encoding
23
print(response.encoding)         # utf-8
24

25
# Cookies
26
print(response.cookies)          # Response cookies
27

28
# History (redirects)
29
print(response.history)          # List of redirects
30

31
# Time
32
print(response.elapsed)          # Time taken
33
print(response.elapsed.total_seconds())

1
import requests
2

3
response = requests.get("https://api.github.com/users/torvalds")
4

5
# Parse JSON response
6
data = response.json()
7

8
print(data["name"])          # Linus Torvalds
9
print(data["public_repos"])  # Number of repos
10
print(data["followers"])     # Followers count

1
import requests
2

3
data = {
4
    "title": "Hello",
5
    "body": "World",
6
    "userId": 1
7
}
8

9
# Method 1: Using json parameter
10
response = requests.post(
11
    "https://jsonplaceholder.typicode.com/posts",
12
    json=data
13
)
14

15
# Method 2: Manual (sets Content-Type automatically with json=)
16
import json
17
response = requests.post(
18
    "https://jsonplaceholder.typicode.com/posts",
19
    data=json.dumps(data),
20
    headers={"Content-Type": "application/json"}
21
)
22

23
print(response.json())

1
import requests
2
import json
3

4
response = requests.get("https://api.github.com/users/torvalds")
5
data = response.json()
6

7
# Pretty print
8
print(json.dumps(data, indent=2))

1
import requests
2

3
data = {
4
    "username": "admin",
5
    "password": "secret"
6
}
7

8
response = requests.post("https://httpbin.org/post", data=data)
9
print(response.json()["form"])

1
import requests
2

3
data = {
4
    "name": "Ahmed",
5
    "skills": ["python", "hacking"]
6
}
7

8
response = requests.post("https://httpbin.org/post", json=data)
9
print(response.json()["json"])

1
import requests
2

3
# Send raw text
4
response = requests.post(
5
    "https://httpbin.org/post",
6
    data="raw text data",
7
    headers={"Content-Type": "text/plain"}
8
)
9

10
# Send raw bytes
11
response = requests.post(
12
    "https://httpbin.org/post",
13
    data=b"binary data"
14
)

1
import requests
2

3
# Open file in binary mode
4
files = {
5
    "file": open("document.pdf", "rb")
6
}
7

8
response = requests.post("https://httpbin.org/post", files=files)
9
print(response.json()["files"])

1
import requests
2

3
files = {
4
    "file": ("custom_name.pdf", open("document.pdf", "rb"), "application/pdf")
5
}
6

7
response = requests.post("https://httpbin.org/post", files=files)

1
import requests
2

3
files = [
4
    ("files", ("file1.txt", open("file1.txt", "rb"))),
5
    ("files", ("file2.txt", open("file2.txt", "rb"))),
6
]
7

8
response = requests.post("https://httpbin.org/post", files=files)

1
import requests
2

3
files = {"file": open("image.jpg", "rb")}
4
data = {"description": "My photo", "tags": "vacation"}
5

6
response = requests.post(
7
    "https://httpbin.org/post",
8
    files=files,
9
    data=data
10
)

1
import requests
2

3
url = "https://example.com/image.jpg"
4
response = requests.get(url)
5

6
with open("downloaded_image.jpg", "wb") as f:
7
    f.write(response.content)
8

9
print("Downloaded!")

1
import requests
2

3
url = "https://example.com/large_file.zip"
4

5
with requests.get(url, stream=True) as response:
6
    response.raise_for_status()
7

8
    total_size = int(response.headers.get("Content-Length", 0))
9
    downloaded = 0
10

11
    with open("large_file.zip", "wb") as f:
12
        for chunk in response.iter_content(chunk_size=8192):
13
            f.write(chunk)
14
            downloaded += len(chunk)
15

16
            # Progress
17
            if total_size:
18
                percent = (downloaded / total_size) * 100
19
                print(f"\rDownloading: {percent:.1f}%", end="")
20

21
print("\nDownload complete!")

1
import requests
2
from tqdm import tqdm
3

4
url = "https://example.com/file.zip"
5

6
response = requests.get(url, stream=True)
7
total_size = int(response.headers.get("Content-Length", 0))
8

9
with open("file.zip", "wb") as f:
10
    with tqdm(total=total_size, unit="B", unit_scale=True) as pbar:
11
        for chunk in response.iter_content(chunk_size=8192):
12
            f.write(chunk)
13
            pbar.update(len(chunk))

1
import requests
2

3
# Create session
4
session = requests.Session()
5

6
# Login
7
login_data = {"username": "admin", "password": "secret"}
8
session.post("https://example.com/login", data=login_data)
9

10
# Subsequent requests use same cookies
11
response = session.get("https://example.com/dashboard")
12
response = session.get("https://example.com/profile")
13

14
# Close session
15
session.close()

1
import requests
2

3
session = requests.Session()
4

5
# Set default headers for all requests
6
session.headers.update({
7
    "User-Agent": "MyApp/1.0",
8
    "Authorization": "Bearer my_token"
9
})
10

11
# All requests will use these headers
12
response = session.get("https://httpbin.org/headers")
13
print(response.json())

1
import requests
2

3
with requests.Session() as session:
4
    session.headers["Authorization"] = "Bearer token123"
5

6
    response = session.get("https://api.example.com/data")
7
    # Session automatically closed after this block

1
import requests
2

3
response = requests.get("https://httpbin.org/cookies/set?name=Ahmed")
4
print(response.cookies)
5

6
# Get specific cookie
7
for cookie in response.cookies:
8
    print(f"{cookie.name}: {cookie.value}")

1
import requests
2

3
# Method 1: Dictionary
4
cookies = {
5
    "session_id": "abc123",
6
    "user": "Ahmed"
7
}
8
response = requests.get("https://httpbin.org/cookies", cookies=cookies)
9
print(response.json())
10

11
# Method 2: CookieJar
12
jar = requests.cookies.RequestsCookieJar()
13
jar.set("session_id", "abc123", domain="httpbin.org")
14
response = requests.get("https://httpbin.org/cookies", cookies=jar)

1
import requests
2

3
session = requests.Session()
4

5
# Set cookies
6
session.cookies.set("token", "secret123")
7

8
response = session.get("https://httpbin.org/cookies")
9
print(response.json())

1
import requests
2
from requests.auth import HTTPBasicAuth
3

4
# Method 1
5
response = requests.get(
6
    "https://httpbin.org/basic-auth/user/pass",
7
    auth=HTTPBasicAuth("user", "pass")
8
)
9

10
# Method 2 (shortcut)
11
response = requests.get(
12
    "https://httpbin.org/basic-auth/user/pass",
13
    auth=("user", "pass")
14
)
15

16
print(response.status_code)  # 200

1
import requests
2
from requests.auth import HTTPDigestAuth
3

4
response = requests.get(
5
    "https://httpbin.org/digest-auth/auth/user/pass",
6
    auth=HTTPDigestAuth("user", "pass")
7
)

1
import requests
2

3
headers = {
4
    "Authorization": "Bearer your_access_token"
5
}
6

7
response = requests.get("https://api.example.com/data", headers=headers)

1
import requests
2

3
# In headers
4
headers = {"X-API-Key": "your_api_key"}
5
response = requests.get("https://api.example.com/data", headers=headers)
6

7
# In query params
8
params = {"api_key": "your_api_key"}
9
response = requests.get("https://api.example.com/data", params=params)

1
import requests
2

3
try:
4
    # Timeout in seconds
5
    response = requests.get("https://httpbin.org/delay/5", timeout=3)
6
except requests.Timeout:
7
    print("Request timed out!")

1
import requests
2

3
# (connect_timeout, read_timeout)
4
response = requests.get(
5
    "https://httpbin.org/get",
6
    timeout=(3.05, 27)  # 3.05s to connect, 27s to read
7
)

1
import requests
2
from requests.adapters import HTTPAdapter
3
from urllib3.util.retry import Retry
4

5
# Configure retry strategy
6
retry_strategy = Retry(
7
    total=3,                    # Total retries
8
    backoff_factor=1,           # Wait 1, 2, 4 seconds between retries
9
    status_forcelist=[500, 502, 503, 504]  # Retry on these status codes
10
)
11

12
# Create session with retry
13
session = requests.Session()
14
adapter = HTTPAdapter(max_retries=retry_strategy)
15
session.mount("http://", adapter)
16
session.mount("https://", adapter)
17

18
# Use session
19
response = session.get("https://example.com/api")

1
import requests
2
import urllib3
3

4
# Disable warnings
5
urllib3.disable_warnings()
6

7
# Skip SSL verification
8
response = requests.get("https://self-signed.badssl.com/", verify=False)

1
import requests
2

3
# Use custom CA bundle
4
response = requests.get("https://example.com", verify="/path/to/ca-bundle.crt")
5

6
# Use client certificate
7
response = requests.get(
8
    "https://example.com",
9
    cert=("/path/to/client.cert", "/path/to/client.key")
10
)

1
import requests
2

3
proxies = {
4
    "http": "http://proxy.example.com:8080",
5
    "https": "http://proxy.example.com:8080"
6
}
7

8
response = requests.get("https://httpbin.org/ip", proxies=proxies)
9
print(response.json())

1
# pip install requests[socks]
2
import requests
3

4
proxies = {
5
    "http": "socks5://127.0.0.1:9050",
6
    "https": "socks5://127.0.0.1:9050"
7
}
8

9
response = requests.get("https://httpbin.org/ip", proxies=proxies)

1
import requests
2

3
proxies = {
4
    "http": "http://user:password@proxy.example.com:8080",
5
    "https": "http://user:password@proxy.example.com:8080"
6
}
7

8
response = requests.get("https://httpbin.org/ip", proxies=proxies)

1
import requests
2

3
try:
4
    response = requests.get("https://example.com", timeout=5)
5
    response.raise_for_status()  # Raise error for 4xx/5xx
6

7
    data = response.json()
8

9
except requests.ConnectionError:
10
    print("Failed to connect!")
11

12
except requests.Timeout:
13
    print("Request timed out!")
14

15
except requests.HTTPError as e:
16
    print(f"HTTP Error: {e}")
17

18
except requests.RequestException as e:
19
    print(f"Request failed: {e}")
20

21
except ValueError:
22
    print("Invalid JSON response")

1
import requests
2

3
response = requests.get("https://api.example.com/data")
4

5
if response.ok:  # status_code < 400
6
    data = response.json()
7
    print(data)
8
else:
9
    print(f"Error: {response.status_code}")
10
    print(response.text)

1
import requests
2
from bs4 import BeautifulSoup
3

4
def scrape_website(url):
5
    headers = {
6
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/91.0"
7
    }
8

9
    response = requests.get(url, headers=headers)
10
    soup = BeautifulSoup(response.text, "html.parser")
11

12
    # Get all links
13
    links = []
14
    for a in soup.find_all("a", href=True):
15
        links.append(a["href"])
16

17
    return links
18

19
links = scrape_website("https://example.com")
20
for link in links[:10]:
21
    print(link)

1
import requests
2

3
class GitHubAPI:
4
    BASE_URL = "https://api.github.com"
5

6
    def __init__(self, token=None):
7
        self.session = requests.Session()
8
        if token:
9
            self.session.headers["Authorization"] = f"token {token}"
10

11
    def get_user(self, username):
12
        response = self.session.get(f"{self.BASE_URL}/users/{username}")
13
        return response.json()
14

15
    def get_repos(self, username):
16
        response = self.session.get(f"{self.BASE_URL}/users/{username}/repos")
17
        return response.json()
18

19
    def search_repos(self, query):
20
        params = {"q": query}
21
        response = self.session.get(f"{self.BASE_URL}/search/repositories", params=params)
22
        return response.json()
23

24
# Usage
25
github = GitHubAPI()
26
user = github.get_user("torvalds")
27
print(f"Name: {user['name']}")
28
print(f"Followers: {user['followers']}")

1
import requests
2
import time
3

4
def check_websites(urls):
5
    results = []
6

7
    for url in urls:
8
        try:
9
            start = time.time()
10
            response = requests.get(url, timeout=10)
11
            elapsed = time.time() - start
12

13
            results.append({
14
                "url": url,
15
                "status": response.status_code,
16
                "time": round(elapsed, 2),
17
                "ok": response.ok
18
            })
19

20
        except requests.RequestException as e:
21
            results.append({
22
                "url": url,
23
                "status": "ERROR",
24
                "time": None,
25
                "ok": False,
26
                "error": str(e)
27
            })
28

29
    return results
30

31
# Check websites
32
websites = [
33
    "https://google.com",
34
    "https://github.com",
35
    "https://python.org",
36
    "https://nonexistent.invalid"
37
]
38

39
results = check_websites(websites)
40
for r in results:
41
    status = "✓" if r["ok"] else "✗"
42
    print(f"{status} {r['url']:30} - {r['status']} ({r['time']}s)")

1
import requests
2

3
def login_to_site(url, username, password):
4
    session = requests.Session()
5

6
    # Get login page (for CSRF token)
7
    login_page = session.get(url)
8

9
    # Parse CSRF token if needed (example)
10
    # from bs4 import BeautifulSoup
11
    # soup = BeautifulSoup(login_page.text, "html.parser")
12
    # csrf = soup.find("input", {"name": "csrf_token"})["value"]
13

14
    # Submit login form
15
    login_data = {
16
        "username": username,
17
        "password": password,
18
        # "csrf_token": csrf
19
    }
20

21
    response = session.post(url, data=login_data)
22

23
    if "Welcome" in response.text or response.url != url:
24
        print("[+] Login successful!")
25
        return session
26
    else:
27
        print("[-] Login failed!")
28
        return None
29

30
# Example usage
31
# session = login_to_site("https://example.com/login", "admin", "password")

1
import requests
2
from concurrent.futures import ThreadPoolExecutor
3

4
def check_path(base_url, path):
5
    url = f"{base_url}/{path}"
6
    try:
7
        response = requests.get(url, timeout=5)
8
        if response.status_code == 200:
9
            print(f"[+] Found: {url}")
10
            return url
11
        elif response.status_code == 403:
12
            print(f"[!] Forbidden: {url}")
13
    except:
14
        pass
15
    return None
16

17
def dir_brute(base_url, wordlist_path, threads=10):
18
    with open(wordlist_path) as f:
19
        paths = [line.strip() for line in f]
20

21
    print(f"[*] Starting brute force on {base_url}")
22
    print(f"[*] Wordlist: {len(paths)} paths")
23

24
    found = []
25
    with ThreadPoolExecutor(max_workers=threads) as executor:
26
        results = executor.map(lambda p: check_path(base_url, p), paths)
27
        found = [r for r in results if r]
28

29
    print(f"\n[*] Found {len(found)} paths")
30
    return found
31

32
# Example usage
33
# dir_brute("https://example.com", "wordlist.txt")