Initial commit: 25 scripts Python sauvegardés

Date: 2026-08-28 11:43:34 EDT
This commit is contained in:
2026-08-28 11:43:34 -04:00
parent 0ba9701e0f
commit 994ea86d47
204 changed files with 70104 additions and 2 deletions
@@ -0,0 +1,59 @@
# Copyright (c) 2020-202x The virtualenv developers
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
Activate virtualenv for current interpreter:
import runpy
runpy.run_path(this_file)
This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
""" # noqa: D415
from __future__ import annotations
import os
import site
import sys
try:
abs_file = os.path.abspath(__file__)
except NameError as exc:
msg = "You must use import runpy; runpy.run_path(this_file)"
raise AssertionError(msg) from exc
bin_dir = os.path.dirname(abs_file)
base = bin_dir[: -len("bin") - 1] # strip away the bin part from the __file__, plus the path separator
# prepend bin to PATH (this file is inside the bin directory)
os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)])
os.environ["VIRTUAL_ENV"] = base # virtual env is right above bin directory
os.environ["VIRTUAL_ENV_PROMPT"] = "" or os.path.basename(base) # noqa: SIM222
# add the virtual environments libraries to the host python import mechanism
prev_length = len(sys.path)
for lib in "../lib/python3.11/site-packages".split(os.pathsep):
path = os.path.realpath(os.path.join(bin_dir, lib))
site.addsitedir(path)
sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
sys.real_prefix = sys.prefix
sys.prefix = base
@@ -0,0 +1,101 @@
"""Patches that are applied at runtime to the virtual environment."""
import os
import sys
VIRTUALENV_PATCH_FILE = os.path.join(__file__)
def patch_dist(dist):
"""
Distutils allows user to configure some arguments via a configuration file:
https://docs.python.org/3.11/install/index.html#distutils-configuration-files.
Some of this arguments though don't make sense in context of the virtual environment files, let's fix them up.
""" # noqa: D205
# we cannot allow some install config as that would get packages installed outside of the virtual environment
old_parse_config_files = dist.Distribution.parse_config_files
def parse_config_files(self, *args, **kwargs):
result = old_parse_config_files(self, *args, **kwargs)
install = self.get_option_dict("install")
if "prefix" in install: # the prefix governs where to install the libraries
install["prefix"] = VIRTUALENV_PATCH_FILE, os.path.abspath(sys.prefix)
for base in ("purelib", "platlib", "headers", "scripts", "data"):
key = f"install_{base}"
if key in install: # do not allow global configs to hijack venv paths
install.pop(key, None)
return result
dist.Distribution.parse_config_files = parse_config_files
# Import hook that patches some modules to ignore configuration values that break package installation in case
# of virtual environments.
_DISTUTILS_PATCH = "distutils.dist", "setuptools.dist"
# https://docs.python.org/3/library/importlib.html#setting-up-an-importer
class _Finder:
"""A meta path finder that allows patching the imported distutils modules."""
fullname = None
# lock[0] is threading.Lock(), but initialized lazily to avoid importing threading very early at startup,
# because there are gevent-based applications that need to be first to import threading by themselves.
# See https://github.com/pypa/virtualenv/issues/1895 for details.
lock = [] # noqa: RUF012
def find_spec(self, fullname, path, target=None): # noqa: ARG002
if fullname in _DISTUTILS_PATCH and self.fullname is None:
# initialize lock[0] lazily
if len(self.lock) == 0:
import threading
lock = threading.Lock()
# there is possibility that two threads T1 and T2 are simultaneously running into find_spec,
# observing .lock as empty, and further going into hereby initialization. However due to the GIL,
# list.append() operation is atomic and this way only one of the threads will "win" to put the lock
# - that every thread will use - into .lock[0].
# https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe
self.lock.append(lock)
from functools import partial
from importlib.util import find_spec
with self.lock[0]:
self.fullname = fullname
try:
spec = find_spec(fullname, path)
if spec is not None:
# https://www.python.org/dev/peps/pep-0451/#how-loading-will-work
is_new_api = hasattr(spec.loader, "exec_module")
func_name = "exec_module" if is_new_api else "load_module"
old = getattr(spec.loader, func_name)
func = self.exec_module if is_new_api else self.load_module
if old is not func:
try: # noqa: SIM105
setattr(spec.loader, func_name, partial(func, old))
except AttributeError:
pass # C-Extension loaders are r/o such as zipimporter with <3.7
return spec
finally:
self.fullname = None
return None
@staticmethod
def exec_module(old, module):
old(module)
if module.__name__ in _DISTUTILS_PATCH:
patch_dist(module)
@staticmethod
def load_module(old, name):
module = old(name)
if module.__name__ in _DISTUTILS_PATCH:
patch_dist(module)
return module
sys.meta_path.insert(0, _Finder())
@@ -0,0 +1,4 @@
from .core import contents, where
__all__ = ["contents", "where"]
__version__ = "2026.07.22"
@@ -0,0 +1,12 @@
import argparse
from certifi import contents, where
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--contents", action="store_true")
args = parser.parse_args()
if args.contents:
print(contents())
else:
print(where())
@@ -0,0 +1,83 @@
"""
certifi.py
~~~~~~~~~~
This module returns the installation location of cacert.pem or its contents.
"""
import sys
import atexit
def exit_cacert_ctx() -> None:
_CACERT_CTX.__exit__(None, None, None) # type: ignore[union-attr]
if sys.version_info >= (3, 11):
from importlib.resources import as_file, files
_CACERT_CTX = None
_CACERT_PATH = None
def where() -> str:
# This is slightly terrible, but we want to delay extracting the file
# in cases where we're inside of a zipimport situation until someone
# actually calls where(), but we don't want to re-extract the file
# on every call of where(), so we'll do it once then store it in a
# global variable.
global _CACERT_CTX
global _CACERT_PATH
if _CACERT_PATH is None:
# This is slightly janky, the importlib.resources API wants you to
# manage the cleanup of this file, so it doesn't actually return a
# path, it returns a context manager that will give you the path
# when you enter it and will do any cleanup when you leave it. In
# the common case of not needing a temporary file, it will just
# return the file system location and the __exit__() is a no-op.
#
# We also have to hold onto the actual context manager, because
# it will do the cleanup whenever it gets garbage collected, so
# we will also store that at the global level as well.
_CACERT_CTX = as_file(files("certifi").joinpath("cacert.pem"))
_CACERT_PATH = str(_CACERT_CTX.__enter__())
atexit.register(exit_cacert_ctx)
return _CACERT_PATH
def contents() -> str:
return files("certifi").joinpath("cacert.pem").read_text(encoding="ascii")
else:
from importlib.resources import path as get_path, read_text
_CACERT_CTX = None
_CACERT_PATH = None
def where() -> str:
# This is slightly terrible, but we want to delay extracting the
# file in cases where we're inside of a zipimport situation until
# someone actually calls where(), but we don't want to re-extract
# the file on every call of where(), so we'll do it once then store
# it in a global variable.
global _CACERT_CTX
global _CACERT_PATH
if _CACERT_PATH is None:
# This is slightly janky, the importlib.resources API wants you
# to manage the cleanup of this file, so it doesn't actually
# return a path, it returns a context manager that will give
# you the path when you enter it and will do any cleanup when
# you leave it. In the common case of not needing a temporary
# file, it will just return the file system location and the
# __exit__() is a no-op.
#
# We also have to hold onto the actual context manager, because
# it will do the cleanup whenever it gets garbage collected, so
# we will also store that at the global level as well.
_CACERT_CTX = get_path("certifi", "cacert.pem")
_CACERT_PATH = str(_CACERT_CTX.__enter__())
atexit.register(exit_cacert_ctx)
return _CACERT_PATH
def contents() -> str:
return read_text("certifi", "cacert.pem", encoding="ascii")
@@ -0,0 +1,18 @@
import os
import unittest
import certifi
class TestCertifi(unittest.TestCase):
def test_cabundle_exists(self) -> None:
assert os.path.exists(certifi.where())
def test_read_contents(self) -> None:
content = certifi.contents()
assert "-----BEGIN CERTIFICATE-----" in content
def test_py_typed_exists(self) -> None:
assert os.path.exists(
os.path.join(os.path.dirname(certifi.__file__), 'py.typed')
)
@@ -0,0 +1,48 @@
"""
Charset-Normalizer
~~~~~~~~~~~~~~
The Real First Universal Charset Detector.
A library that helps you read text from an unknown charset encoding.
Motivated by chardet, This package is trying to resolve the issue by taking a new approach.
All IANA character set names for which the Python core library provides codecs are supported.
Basic usage:
>>> from charset_normalizer import from_bytes
>>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8'))
>>> best_guess = results.best()
>>> str(best_guess)
'Bсеки човек има право на образование. Oбразованието!'
Others methods and usages are available - see the full documentation
at <https://github.com/Ousret/charset_normalizer>.
:copyright: (c) Ahmed TAHRI
:license: MIT, see LICENSE for more details.
"""
from __future__ import annotations
import logging
from .api import from_bytes, from_fp, from_path, is_binary
from .legacy import detect
from .models import CharsetMatch, CharsetMatches
from .utils import set_logging_handler
from .version import VERSION, __version__
__all__ = (
"from_fp",
"from_path",
"from_bytes",
"is_binary",
"detect",
"CharsetMatch",
"CharsetMatches",
"__version__",
"VERSION",
"set_logging_handler",
)
# Attach a NullHandler to the top level logger by default
# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
logging.getLogger("charset_normalizer").addHandler(logging.NullHandler())
@@ -0,0 +1,6 @@
from __future__ import annotations
from .cli import cli_detect
if __name__ == "__main__":
cli_detect()
@@ -0,0 +1,934 @@
from __future__ import annotations
import logging
from functools import lru_cache
from os import PathLike
from typing import BinaryIO
from .cd import (
coherence_ratio,
encoding_languages,
mb_encoding_languages,
merge_coherence_ratios,
)
from .constant import (
IANA_SUPPORTED,
IANA_SUPPORTED_SIMILAR,
TOO_BIG_SEQUENCE,
TOO_SMALL_SEQUENCE,
TRACE,
)
from .md import mess_ratio
from .models import CharsetMatch, CharsetMatches
from .utils import (
any_specified_encoding,
cut_sequence_chunks,
iana_name,
identify_sig_or_bom,
is_multi_byte_encoding,
should_strip_sig_or_bom,
)
logger = logging.getLogger("charset_normalizer")
explain_handler = logging.StreamHandler()
explain_handler.setFormatter(
logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
)
# Pre-compute a reordered encoding list: multibyte first, then single-byte.
# This allows the mb_definitive_match optimization to fire earlier, skipping
# all single-byte encodings for genuine CJK content. Multibyte codecs
# hard-fail (UnicodeDecodeError) on single-byte data almost instantly, so
# testing them first costs negligible time for non-CJK files.
# Stable sort on a boolean key: multibyte (False) first, IANA order kept
# within each group.
IANA_SUPPORTED_MB_FIRST: list[str] = sorted(
IANA_SUPPORTED, key=lambda encoding: not is_multi_byte_encoding(encoding)
)
def from_bytes(
sequences: bytes | bytearray,
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.2,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Given a raw bytes sequence, return the best possibles charset usable to render str objects.
If there is no results, it is a strong indicator that the source is binary/not text.
By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence.
And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will.
The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page
but never take it for granted. Can improve the performance.
You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that
purpose.
This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32.
By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain'
toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging.
Custom logging format and handler can be set manually.
"""
if not isinstance(sequences, (bytearray, bytes)):
raise TypeError(
"Expected object of type bytes or bytearray, got: {}".format(
type(sequences)
)
)
if explain:
previous_logger_level: int = logger.level
logger.addHandler(explain_handler)
logger.setLevel(TRACE)
length: int = len(sequences)
if length == 0:
logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.")
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")])
if cp_isolation is not None:
logger.log(
TRACE,
"cp_isolation is set. use this flag for debugging purpose. "
"limited list of encoding allowed : %s.",
", ".join(cp_isolation),
)
cp_isolation = [iana_name(cp, False) for cp in cp_isolation]
else:
cp_isolation = []
if cp_exclusion is not None:
logger.log(
TRACE,
"cp_exclusion is set. use this flag for debugging purpose. "
"limited list of encoding excluded : %s.",
", ".join(cp_exclusion),
)
cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion]
else:
cp_exclusion = []
if length <= (chunk_size * steps):
logger.log(
TRACE,
"override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.",
steps,
chunk_size,
length,
)
steps = 1
chunk_size = length
if steps > 1 and length / steps < chunk_size:
chunk_size = int(length / steps)
is_too_small_sequence: bool = len(sequences) < TOO_SMALL_SEQUENCE
is_too_large_sequence: bool = len(sequences) >= TOO_BIG_SEQUENCE
if is_too_small_sequence:
logger.log(
TRACE,
"Trying to detect encoding from a tiny portion of (%i) byte(s).",
length,
)
elif is_too_large_sequence:
logger.log(
TRACE,
"Using lazy str decoding because the payload is quite large, (%i) byte(s).",
length,
)
prioritized_encodings: list[str] = []
specified_encoding: str | None = (
any_specified_encoding(sequences) if preemptive_behaviour else None
)
if specified_encoding is not None:
prioritized_encodings.append(specified_encoding)
logger.log(
TRACE,
"Detected declarative mark in sequence. Priority +1 given for %s.",
specified_encoding,
)
tested: set[str] = set()
tested_but_hard_failure: list[str] = []
tested_but_soft_failure: list[str] = []
soft_failure_skip: set[str] = set()
# Avoid unoptimized RSS usage.
# this cache is mostly interesting for
# local usage. Garbage collected at the
# end. Like it should.
cached_mess_ratio = lru_cache(maxsize=None)(mess_ratio)
cached_coherence_ratio = lru_cache(maxsize=None)(coherence_ratio)
# When a definitive result (chaos=0.0 and good coherence) is found after testing
# the prioritized encodings (ascii, utf_8), we can significantly reduce the remaining
# work. Encodings that target completely different language families (e.g., Cyrillic
# when the definitive match is Latin) are skipped entirely.
# Additionally, for same-family encodings that pass chaos probing, we reuse the
# definitive match's coherence ratios instead of recomputing them — a major savings
# since coherence_ratio accounts for ~30% of total time on slow Latin files.
definitive_match_found: bool = False
definitive_target_languages: set[str] = set()
# After the definitive match fires, we cap the number of additional same-family
# single-byte encodings that pass chaos probing. Once we've accumulated enough
# good candidates (N), further same-family SB encodings are unlikely to produce
# a better best() result and just waste mess_ratio + coherence_ratio time.
# The first encoding to trigger the definitive match is NOT counted (it's already in).
post_definitive_sb_success_count: int = 0
POST_DEFINITIVE_SB_CAP: int = 7
# When a non-UTF multibyte encoding passes chaos probing with significant multibyte
# content (decoded length < 98% of raw length), skip all remaining single-byte encodings.
# Rationale: multi-byte decoders (CJK) have strict byte-sequence validation — if they
# decode without error AND pass chaos probing with substantial multibyte content, the
# data is genuinely multibyte encoded. Single-byte encodings will always decode (every
# byte maps to something) but waste time on mess_ratio before failing.
# The 98% threshold prevents false triggers on files that happen to have a few valid
# multibyte pairs (e.g., cp424/_ude_1.txt where big5 decodes with 99% ratio).
mb_definitive_match_found: bool = False
fallback_ascii: CharsetMatch | None = None
fallback_u8: CharsetMatch | None = None
fallback_specified: CharsetMatch | None = None
results: CharsetMatches = CharsetMatches()
early_stop_results: CharsetMatches = CharsetMatches()
sig_encoding, sig_payload = identify_sig_or_bom(sequences)
if sig_encoding is not None:
prioritized_encodings.insert(0, sig_encoding)
logger.log(
TRACE,
"Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.",
len(sig_payload),
sig_encoding,
)
prioritized_encodings.append("ascii")
if "utf_8" not in prioritized_encodings:
prioritized_encodings.append("utf_8")
for encoding_iana in prioritized_encodings + IANA_SUPPORTED_MB_FIRST:
if cp_isolation and encoding_iana not in cp_isolation:
continue
if cp_exclusion and encoding_iana in cp_exclusion:
continue
if encoding_iana in tested:
continue
tested.add(encoding_iana)
decoded_payload: str | None = None
bom_or_sig_available: bool = sig_encoding == encoding_iana
strip_sig_or_bom: bool = bom_or_sig_available and should_strip_sig_or_bom(
encoding_iana
)
if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available:
logger.log(
TRACE,
"Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.",
encoding_iana,
)
continue
if encoding_iana in {"utf_7"} and not bom_or_sig_available:
logger.log(
TRACE,
"Encoding %s won't be tested as-is because detection is unreliable without BOM/SIG.",
encoding_iana,
)
continue
# Skip encodings similar to ones that already soft-failed (high mess ratio).
# Checked BEFORE the expensive decode attempt.
if encoding_iana in soft_failure_skip:
logger.log(
TRACE,
"%s is deemed too similar to a code page that was already considered unsuited. Continuing!",
encoding_iana,
)
continue
try:
is_multi_byte_decoder: bool = is_multi_byte_encoding(encoding_iana)
except (ModuleNotFoundError, ImportError): # Defensive:
logger.log(
TRACE,
"Encoding %s does not provide an IncrementalDecoder",
encoding_iana,
)
continue
# When we've already found a definitive match (chaos=0.0 with good coherence)
# after testing the prioritized encodings, skip encodings that target
# completely different language families. This avoids running expensive
# mess_ratio + coherence_ratio on clearly unrelated candidates (e.g., Cyrillic
# when the definitive match is Latin-based).
if definitive_match_found:
if not is_multi_byte_decoder:
enc_languages = set(encoding_languages(encoding_iana))
else:
enc_languages = set(mb_encoding_languages(encoding_iana))
if not enc_languages.intersection(definitive_target_languages):
logger.log(
TRACE,
"Skipping %s: definitive match already found, this encoding targets different languages (%s vs %s).",
encoding_iana,
enc_languages,
definitive_target_languages,
)
continue
# After the definitive match, cap the number of additional same-family
# single-byte encodings that pass chaos probing. This avoids testing the
# tail of rare, low-value same-family encodings (mac_iceland, cp860, etc.)
# that almost never change best() but each cost ~1-2ms of mess_ratio + coherence.
if (
definitive_match_found
and not is_multi_byte_decoder
and post_definitive_sb_success_count >= POST_DEFINITIVE_SB_CAP
):
logger.log(
TRACE,
"Skipping %s: already accumulated %d same-family results after definitive match (cap=%d).",
encoding_iana,
post_definitive_sb_success_count,
POST_DEFINITIVE_SB_CAP,
)
continue
# When a multibyte encoding with significant multibyte content has already
# passed chaos probing, skip all single-byte encodings. They will either fail
# chaos probing (wasting mess_ratio time) or produce inferior results.
if mb_definitive_match_found and not is_multi_byte_decoder:
logger.log(
TRACE,
"Skipping single-byte %s: multi-byte definitive match already found.",
encoding_iana,
)
continue
# Single-byte candidates of regular size defer the expensive whole
# payload decode until after chunk probing: single-byte codecs are
# stateless (1 byte == 1 char) so decoding chunk slices is provably
# identical to slicing the decoded payload, and candidates rejected
# by chaos probing (the common case) never pay the full decode nor
# the payload hash.
deferred_decoding: bool = (
not is_multi_byte_decoder and not is_too_large_sequence
)
try:
if is_too_large_sequence and not is_multi_byte_decoder:
str(
(
sequences[: int(50e4)]
if not strip_sig_or_bom
else sequences[len(sig_payload) : int(50e4)]
),
encoding=encoding_iana,
)
elif not deferred_decoding:
# UTF-7 BOM is encoded in modified Base64 whose byte boundary
# can overlap with the next character. Stripping raw SIG bytes
# before decoding may leave stray bytes that decode as garbage.
# Decode the full sequence and remove the leading BOM char instead.
# see https://github.com/jawah/charset_normalizer/issues/718
# and https://github.com/jawah/charset_normalizer/issues/716
if encoding_iana == "utf_7" and bom_or_sig_available:
decoded_payload = str(
sequences,
encoding=encoding_iana,
)
if decoded_payload and decoded_payload[0] == "\ufeff":
decoded_payload = decoded_payload[1:]
else:
decoded_payload = str(
(
sequences
if not strip_sig_or_bom
else sequences[len(sig_payload) :]
),
encoding=encoding_iana,
)
except (UnicodeDecodeError, LookupError) as e:
if not isinstance(e, LookupError):
logger.log(
TRACE,
"Code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
tested_but_hard_failure.append(encoding_iana)
continue
r_ = range(
0 if not bom_or_sig_available else len(sig_payload),
length,
int(length / steps),
)
multi_byte_bonus: bool = (
is_multi_byte_decoder
and decoded_payload is not None
and len(decoded_payload) < length
)
if multi_byte_bonus:
logger.log(
TRACE,
"Code page %s is a multi byte encoding table and it appear that at least one character "
"was encoded using n-bytes.",
encoding_iana,
)
max_chunk_gave_up: int = int(len(r_) / 4)
max_chunk_gave_up = max(max_chunk_gave_up, 2)
early_stop_count: int = 0
lazy_str_hard_failure = False
md_chunks: list[str] = []
md_ratios = []
try:
for chunk in cut_sequence_chunks(
sequences,
encoding_iana,
r_,
chunk_size,
bom_or_sig_available,
strip_sig_or_bom,
sig_payload,
is_multi_byte_decoder,
decoded_payload,
deferred_decoding,
):
md_chunks.append(chunk)
md_ratios.append(
cached_mess_ratio(
chunk,
threshold,
explain and 1 <= len(cp_isolation) <= 2,
)
)
if md_ratios[-1] >= threshold:
early_stop_count += 1
if (early_stop_count >= max_chunk_gave_up) or (
bom_or_sig_available and not strip_sig_or_bom
):
break
except (
UnicodeDecodeError,
LookupError,
) as e: # Lazy str loading may have missed something there
if deferred_decoding:
# Deferred single-byte validation failed on a chunk (or the
# codec is unavailable on this interpreter build): identical
# outcome and bookkeeping to the eager full-decode failure.
logger.log(
TRACE,
"Code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
tested_but_hard_failure.append(encoding_iana)
continue
logger.log(
TRACE,
"LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
early_stop_count = max_chunk_gave_up
lazy_str_hard_failure = True
mean_mess_ratio: float = sum(md_ratios) / len(md_ratios) if md_ratios else 0.0
# We might want to check the sequence again with the whole content,
# but only if initial MD tests passed.
if (
not lazy_str_hard_failure
and is_too_large_sequence
and not is_multi_byte_decoder
and mean_mess_ratio < threshold
and early_stop_count < max_chunk_gave_up
):
try:
sequences[int(50e3) :].decode(encoding_iana, errors="strict")
except UnicodeDecodeError as e:
logger.log(
TRACE,
"LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
tested_but_hard_failure.append(encoding_iana)
continue
if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up:
tested_but_soft_failure.append(encoding_iana)
if encoding_iana in IANA_SUPPORTED_SIMILAR:
soft_failure_skip.update(IANA_SUPPORTED_SIMILAR[encoding_iana])
logger.log(
TRACE,
"%s was excluded because of initial chaos probing. Gave up %i time(s). "
"Computed mean chaos is %f %%.",
encoding_iana,
early_stop_count,
round(mean_mess_ratio * 100, ndigits=3),
)
# Preparing those fallbacks in case we got nothing.
if (
enable_fallback
and encoding_iana
in ["ascii", "utf_8", specified_encoding, "utf_16", "utf_32"]
and not lazy_str_hard_failure
):
# Always fully decode payload before.
# We've missed a UnicodeDecodeError proof
# while issuing release 3.4.8
# see https://github.com/jawah/charset_normalizer/issues/771
if decoded_payload is None:
try:
decoded_payload = str(
(
sequences
if not strip_sig_or_bom
else sequences[len(sig_payload) :]
),
encoding=encoding_iana,
)
except (UnicodeDecodeError, LookupError):
logger.log(
TRACE,
"%s does not decode the whole payload: fallback entry withheld.",
encoding_iana,
)
continue
if is_too_large_sequence:
# Don't retain huge payload in RAM.
decoded_payload = None
fallback_entry = CharsetMatch(
sequences,
encoding_iana,
threshold,
bom_or_sig_available,
[],
decoded_payload,
preemptive_declaration=specified_encoding,
)
if encoding_iana == specified_encoding:
fallback_specified = fallback_entry
elif encoding_iana == "ascii":
fallback_ascii = fallback_entry
else:
fallback_u8 = fallback_entry
continue
if deferred_decoding:
# The candidate passed chaos probing: perform the whole payload
# decode (validation + payload reuse) that was deferred earlier.
try:
decoded_payload = str(
(
sequences
if not strip_sig_or_bom
else sequences[len(sig_payload) :]
),
encoding=encoding_iana,
)
except (UnicodeDecodeError, LookupError) as e:
logger.log(
TRACE,
"Code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
tested_but_hard_failure.append(encoding_iana)
continue
logger.log(
TRACE,
"%s passed initial chaos probing. Mean measured chaos is %f %%",
encoding_iana,
round(mean_mess_ratio * 100, ndigits=3),
)
if not is_multi_byte_decoder:
target_languages: list[str] = encoding_languages(encoding_iana)
else:
target_languages = mb_encoding_languages(encoding_iana)
if target_languages:
logger.log(
TRACE,
"%s should target any language(s) of %s",
encoding_iana,
target_languages,
)
cd_ratios = []
# Run coherence detection on all chunks. We previously tried limiting to
# 1-2 chunks for post-definitive encodings to save time, but this caused
# coverage regressions by producing unrepresentative coherence scores.
# The SB cap and language-family skip optimizations provide sufficient
# speedup without sacrificing coherence accuracy.
if encoding_iana != "ascii":
# We shall skip the CD when its about ASCII
# Most of the time its not relevant to run "language-detection" on it.
lg_inclusion: str | None = (
",".join(target_languages) if target_languages else None
)
for chunk in md_chunks:
chunk_languages = cached_coherence_ratio(
chunk,
language_threshold,
lg_inclusion,
)
cd_ratios.append(chunk_languages)
cd_ratios_merged = merge_coherence_ratios(cd_ratios)
if cd_ratios_merged:
logger.log(
TRACE,
"We detected language %s using %s",
cd_ratios_merged,
encoding_iana,
)
current_match = CharsetMatch(
sequences,
encoding_iana,
mean_mess_ratio,
bom_or_sig_available,
cd_ratios_merged,
(
decoded_payload
if (
not is_too_large_sequence
or encoding_iana in [specified_encoding, "ascii", "utf_8"]
)
else None
),
preemptive_declaration=specified_encoding,
)
results.append(current_match)
# Count post-definitive same-family SB successes for the early termination cap.
# Only count low-mess encodings (< 2%) toward the cap. High-mess encodings are
# marginal results that shouldn't prevent better-quality candidates from being
# tested. For example, iso8859_4 (mess=0%) should not be skipped just because
# 7 high-mess Latin encodings (cp1252 at 8%, etc.) were tried first.
if (
definitive_match_found
and not is_multi_byte_decoder
and mean_mess_ratio < 0.02
):
post_definitive_sb_success_count += 1
if (
encoding_iana in [specified_encoding, "ascii", "utf_8"]
and mean_mess_ratio < 0.1
):
# If md says nothing to worry about, then... stop immediately!
if mean_mess_ratio == 0.0:
logger.debug(
"Encoding detection: %s is most likely the one.",
current_match.encoding,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([current_match])
early_stop_results.append(current_match)
if (
len(early_stop_results)
and (specified_encoding is None or specified_encoding in tested)
and "ascii" in tested
and "utf_8" in tested
):
probable_result = early_stop_results.best()
assert probable_result is not None
logger.debug(
"Encoding detection: %s is most likely the one.",
probable_result.encoding,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([probable_result])
# Once we find a result with good coherence (>= 0.5) after testing the
# prioritized encodings (ascii, utf_8), activate "definitive mode": skip
# encodings that target completely different language families. This avoids
# running expensive mess_ratio + coherence_ratio on clearly unrelated
# candidates (e.g., Cyrillic encodings when the match is Latin-based).
# We require coherence >= 0.5 to avoid false positives (e.g., cp1251 decoding
# Hebrew text with 0.0 chaos but wrong language detection at coherence 0.33).
if not definitive_match_found and not is_multi_byte_decoder:
best_coherence = (
max((v for _, v in cd_ratios_merged), default=0.0)
if cd_ratios_merged
else 0.0
)
if best_coherence >= 0.5 and "ascii" in tested and "utf_8" in tested:
definitive_match_found = True
definitive_target_languages.update(target_languages)
logger.log(
TRACE,
"Definitive match found: %s (chaos=%.3f, coherence=%.2f). Encodings targeting different language families will be skipped.",
encoding_iana,
mean_mess_ratio,
best_coherence,
)
# When a non-UTF multibyte encoding passes chaos probing with significant
# multibyte content (decoded < 98% of raw), activate mb_definitive_match.
# This skips all remaining single-byte encodings which would either soft-fail
# (running expensive mess_ratio for nothing) or produce inferior results.
if (
not mb_definitive_match_found
and is_multi_byte_decoder
and multi_byte_bonus
and decoded_payload is not None
and len(decoded_payload) < length * 0.98
and encoding_iana
not in {
"utf_8",
"utf_8_sig",
"utf_16",
"utf_16_be",
"utf_16_le",
"utf_32",
"utf_32_be",
"utf_32_le",
"utf_7",
}
and "ascii" in tested
and "utf_8" in tested
):
mb_definitive_match_found = True
logger.log(
TRACE,
"Multi-byte definitive match: %s (chaos=%.3f, decoded=%d/%d=%.1f%%). Single-byte encodings will be skipped.",
encoding_iana,
mean_mess_ratio,
len(decoded_payload),
length,
len(decoded_payload) / length * 100,
)
if encoding_iana == sig_encoding:
logger.debug(
"Encoding detection: %s is most likely the one as we detected a BOM or SIG within "
"the beginning of the sequence.",
encoding_iana,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([results[encoding_iana]])
if len(results) == 0:
if fallback_u8 or fallback_ascii or fallback_specified:
logger.log(
TRACE,
"Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.",
)
if fallback_specified:
logger.debug(
"Encoding detection: %s will be used as a fallback match",
fallback_specified.encoding,
)
results.append(fallback_specified)
elif (
(fallback_u8 and fallback_ascii is None)
or (
fallback_u8
and fallback_ascii
and fallback_u8.fingerprint != fallback_ascii.fingerprint
)
or (fallback_u8 is not None)
):
logger.debug("Encoding detection: utf_8 will be used as a fallback match")
results.append(fallback_u8)
elif fallback_ascii:
logger.debug("Encoding detection: ascii will be used as a fallback match")
results.append(fallback_ascii)
if results:
logger.debug(
"Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.",
results.best().encoding, # type: ignore
len(results) - 1,
)
else:
logger.debug("Encoding detection: Unable to determine any suitable charset.")
if explain:
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return results
def from_fp(
fp: BinaryIO,
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Same thing than the function from_bytes but using a file pointer that is already ready.
Will not close the file pointer.
"""
return from_bytes(
fp.read(),
steps,
chunk_size,
threshold,
cp_isolation,
cp_exclusion,
preemptive_behaviour,
explain,
language_threshold,
enable_fallback,
)
def from_path(
path: str | bytes | PathLike, # type: ignore[type-arg]
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode.
Can raise IOError.
"""
with open(path, "rb") as fp:
return from_fp(
fp,
steps,
chunk_size,
threshold,
cp_isolation,
cp_exclusion,
preemptive_behaviour,
explain,
language_threshold,
enable_fallback,
)
def is_binary(
fp_or_path_or_payload: PathLike | str | BinaryIO | bytes, # type: ignore[type-arg]
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = False,
) -> bool:
"""
Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string.
Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match
are disabled to be stricter around ASCII-compatible but unlikely to be a string.
"""
if isinstance(fp_or_path_or_payload, (str, PathLike)):
guesses = from_path(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
elif isinstance(
fp_or_path_or_payload,
(
bytes,
bytearray,
),
):
guesses = from_bytes(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
else:
guesses = from_fp(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
return not guesses
@@ -0,0 +1,467 @@
from __future__ import annotations
import importlib
from codecs import IncrementalDecoder
from functools import lru_cache
from .constant import (
FREQUENCIES,
KO_NAMES,
LANGUAGE_SUPPORTED_COUNT,
TOO_SMALL_SEQUENCE,
ZH_NAMES,
_FREQUENCIES_SET,
_FREQUENCIES_RANK,
)
from .md import _ASCII_CHAR_INFO, _char_info, is_suspiciously_successive_range
from .models import CoherenceMatches
from .utils import (
is_multi_byte_encoding,
is_unicode_range_secondary,
)
def encoding_unicode_range(iana_name: str) -> list[str]:
"""
Return associated unicode ranges in a single byte code page.
"""
if is_multi_byte_encoding(iana_name):
raise OSError( # Defensive:
"Function not supported on multi-byte code page"
)
decoder = importlib.import_module(f"encodings.{iana_name}").IncrementalDecoder
p: IncrementalDecoder = decoder(errors="ignore")
seen_ranges: dict[str, int] = {}
character_count: int = 0
for i in range(0x40, 0xFF):
chunk: str = p.decode(bytes([i]))
if chunk:
chunk_codepoint = ord(chunk)
character_range: str | None = (
_ASCII_CHAR_INFO[chunk_codepoint].range
if chunk_codepoint < 128
else _char_info(chunk).range
)
if character_range is None:
continue
if not is_unicode_range_secondary(character_range):
if character_range not in seen_ranges:
seen_ranges[character_range] = 0
seen_ranges[character_range] += 1
character_count += 1
return sorted(
[
character_range
for character_range in seen_ranges
if seen_ranges[character_range] / character_count >= 0.15
]
)
def unicode_range_languages(primary_range: str) -> list[str]:
"""
Return inferred languages used with a unicode range.
"""
languages: list[str] = []
for language, characters in FREQUENCIES.items():
for character in characters:
codepoint = ord(character)
info = (
_ASCII_CHAR_INFO[codepoint]
if codepoint < 128
else _char_info(character)
)
if info.range == primary_range:
languages.append(language)
break
return languages
@lru_cache()
def encoding_languages(iana_name: str) -> list[str]:
"""
Single-byte encoding language association. Some code page are heavily linked to particular language(s).
This function does the correspondence.
"""
try:
unicode_ranges: list[str] = encoding_unicode_range(iana_name)
except ImportError: # Defensive: encoding unavailable on this build.
return []
primary_range: str | None = None
for specified_range in unicode_ranges:
if "Latin" not in specified_range:
primary_range = specified_range
break
if primary_range is None:
return ["Latin Based"]
return unicode_range_languages(primary_range)
@lru_cache()
def mb_encoding_languages(iana_name: str) -> list[str]:
"""
Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
This function does the correspondence.
"""
if (
iana_name.startswith("shift_")
or iana_name.startswith("iso2022_jp")
or iana_name.startswith("euc_j")
or iana_name == "cp932"
):
return ["Japanese"]
if iana_name.startswith("gb") or iana_name in ZH_NAMES:
return ["Chinese"]
if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES:
return ["Korean"]
return []
@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT)
def get_target_features(language: str) -> tuple[bool, bool]:
"""
Determine main aspects from a supported language if it contains accents and if is pure Latin.
"""
target_have_accents: bool = False
target_pure_latin: bool = True
for character in FREQUENCIES[language]:
codepoint = ord(character)
info = _ASCII_CHAR_INFO[codepoint] if codepoint < 128 else _char_info(character)
if not target_have_accents and info.accentuated:
target_have_accents = True
if target_pure_latin and not info.latin:
target_pure_latin = False
return target_have_accents, target_pure_latin
def alphabet_languages(
characters: list[str], ignore_non_latin: bool = False
) -> list[str]:
"""
Return associated languages associated to given characters.
"""
languages: list[tuple[str, float]] = []
characters_set: frozenset[str] = frozenset(characters)
source_have_accents = False
for character in characters:
codepoint = ord(character)
info = _ASCII_CHAR_INFO[codepoint] if codepoint < 128 else _char_info(character)
if info.accentuated:
source_have_accents = True
break
for language, language_characters in FREQUENCIES.items():
target_have_accents, target_pure_latin = get_target_features(language)
if ignore_non_latin and not target_pure_latin:
continue
if not target_have_accents and source_have_accents:
continue
character_count: int = len(language_characters)
character_match_count: int = len(_FREQUENCIES_SET[language] & characters_set)
ratio: float = character_match_count / character_count
if ratio >= 0.2:
languages.append((language, ratio))
languages = sorted(languages, key=lambda x: x[1], reverse=True)
return [compatible_language[0] for compatible_language in languages]
def characters_popularity_compare(
language: str, ordered_characters: list[str]
) -> float:
"""
Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
"""
if language not in FREQUENCIES:
raise ValueError(f"{language} not available") # Defensive:
character_approved_count: int = 0
lang_rank: dict[str, int] = _FREQUENCIES_RANK[language]
ordered_characters_count: int = len(ordered_characters)
target_language_characters_count: int = len(FREQUENCIES[language])
large_alphabet: bool = target_language_characters_count > 26
large_alphabet_threshold: float = target_language_characters_count / 3
expected_projection_ratio: float = (
target_language_characters_count / ordered_characters_count
)
# Single pass: characters present in the language vocabulary, as
# (language rank, popularity rank) pairs. The scoring below only ever
# needs ranks, never the characters themselves.
common_lr: list[int] = []
common_orr: list[int] = []
for popularity_rank, character in enumerate(ordered_characters):
language_rank = lang_rank.get(character)
if language_rank is not None:
common_lr.append(language_rank)
common_orr.append(popularity_rank)
for character_rank_in_language, character_rank in zip(common_lr, common_orr):
character_rank_projection: int = int(character_rank * expected_projection_ratio)
if (
not large_alphabet
and abs(character_rank_projection - character_rank_in_language) > 4
):
continue
if (
large_alphabet
and abs(character_rank_projection - character_rank_in_language)
< large_alphabet_threshold
):
character_approved_count += 1
continue
if character_rank_in_language == 0:
# before_match_count is structurally 0 here (no pair can have a
# smaller language rank): the historic "before <= 4" acceptance
# always holds. (The symmetric "after_len == 0" case is
# impossible: language ranks are strictly below the language
# character count, hence after_len >= 1.)
character_approved_count += 1
continue
after_len: int = target_language_characters_count - character_rank_in_language
# Count how many characters appear "before" in both orderings, and
# how many appear "at or after" in both orderings. Both counts grow
# monotonically and the approval thresholds
# (before / rank >= 0.4 or after / after_len >= 0.4) are known
# upfront, expressed below as exact integer comparisons: exit as
# soon as one is crossed.
before_match_count: int = 0
after_match_count: int = 0
for lr_i, orr_i in zip(common_lr, common_orr):
if lr_i < character_rank_in_language:
if orr_i < character_rank:
before_match_count += 1
if 5 * before_match_count >= 2 * character_rank_in_language:
character_approved_count += 1
break
else:
if orr_i >= character_rank:
after_match_count += 1
if 5 * after_match_count >= 2 * after_len:
character_approved_count += 1
break
return character_approved_count / len(ordered_characters)
def alpha_unicode_split(decoded_sequence: str) -> list[str]:
"""
Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
One containing the latin letters and the other hebrew.
"""
layers: dict[str, list[str]] = {}
# Fast path: track single-layer key to skip dict iteration for single-script text.
single_layer_key: str | None = None
multi_layer: bool = False
# Cache the last character_range and its resolved layer to avoid repeated
# is_suspiciously_successive_range calls for consecutive same-range chars.
prev_character_range: str | None = None
prev_layer_target: str | None = None
for character in decoded_sequence:
# Reuse the per-codepoint CharInfo cache: info.alpha and info.range
# are computed with the very same str.isalpha() / unicode_range()
# calls this loop historically made per character occurrence.
codepoint: int = ord(character)
if codepoint < 128:
info = _ASCII_CHAR_INFO[codepoint]
else:
info = _char_info(character)
if not info.alpha:
continue
character_range: str | None = info.range
if character_range is None:
continue
# Fast path: same range as previous character → reuse cached layer target.
if character_range == prev_character_range:
if prev_layer_target is not None:
layers[prev_layer_target].append(character)
continue
layer_target_range: str | None = None
if multi_layer:
for discovered_range in layers:
if not is_suspiciously_successive_range(
discovered_range, character_range
):
layer_target_range = discovered_range
break
elif single_layer_key is not None:
if not is_suspiciously_successive_range(single_layer_key, character_range):
layer_target_range = single_layer_key
if layer_target_range is None:
layer_target_range = character_range
if layer_target_range not in layers:
layers[layer_target_range] = []
if single_layer_key is None:
single_layer_key = layer_target_range
else:
multi_layer = True
layers[layer_target_range].append(character)
# Cache for next iteration
prev_character_range = character_range
prev_layer_target = layer_target_range
return ["".join(chars).lower() for chars in layers.values()]
def merge_coherence_ratios(results: list[CoherenceMatches]) -> CoherenceMatches:
"""
This function merge results previously given by the function coherence_ratio.
The return type is the same as coherence_ratio.
"""
per_language_ratios: dict[str, list[float]] = {}
for result in results:
for sub_result in result:
language, ratio = sub_result
if language not in per_language_ratios:
per_language_ratios[language] = [ratio]
continue
per_language_ratios[language].append(ratio)
merge = [
(
language,
round(
sum(per_language_ratios[language]) / len(per_language_ratios[language]),
4,
),
)
for language in per_language_ratios
]
return sorted(merge, key=lambda x: x[1], reverse=True)
def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches:
"""
We shall NOT return "English—" in CoherenceMatches because it is an alternative
of "English". This function only keeps the best match and remove the em-dash in it.
"""
index_results: dict[str, list[float]] = dict()
for result in results:
language, ratio = result
no_em_name: str = language.replace("—", "")
if no_em_name not in index_results:
index_results[no_em_name] = []
index_results[no_em_name].append(ratio)
if any(len(index_results[e]) > 1 for e in index_results):
filtered_results: CoherenceMatches = []
for language in index_results:
filtered_results.append((language, max(index_results[language])))
return filtered_results
return results
def coherence_ratio(
decoded_sequence: str, threshold: float = 0.1, lg_inclusion: str | None = None
) -> CoherenceMatches:
"""
Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
A layer = Character extraction by alphabets/ranges.
"""
results: list[tuple[str, float]] = []
ignore_non_latin: bool = False
sufficient_match_count: int = 0
lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else []
if "Latin Based" in lg_inclusion_list:
ignore_non_latin = True
lg_inclusion_list.remove("Latin Based")
for layer in alpha_unicode_split(decoded_sequence):
# Native counting + stable sort reproduce Counter.most_common()
# ordering exactly (ties keep first-appearance order) without the
# interpreted Counter machinery in the compiled hot path.
char_counts: dict[str, int] = {}
for layer_character in layer:
char_counts[layer_character] = char_counts.get(layer_character, 0) + 1
character_count: int = len(layer)
if character_count <= TOO_SMALL_SEQUENCE:
continue
popular_character_ordered: list[str] = [
item[0]
for item in sorted(
char_counts.items(), key=lambda item: item[1], reverse=True
)
]
for language in lg_inclusion_list or alphabet_languages(
popular_character_ordered, ignore_non_latin
):
ratio: float = characters_popularity_compare(
language, popular_character_ordered
)
if ratio < threshold:
continue
elif ratio >= 0.8:
sufficient_match_count += 1
results.append((language, round(ratio, 4)))
if sufficient_match_count >= 3:
break
return sorted(
filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True
)
@@ -0,0 +1,8 @@
from __future__ import annotations
from .__main__ import cli_detect, query_yes_no
__all__ = (
"cli_detect",
"query_yes_no",
)
@@ -0,0 +1,363 @@
from __future__ import annotations
import argparse
import sys
import typing
from os.path import abspath, basename, dirname, join, realpath
from platform import python_version
from unicodedata import unidata_version
import charset_normalizer.md as md_module
from charset_normalizer import from_fp
from charset_normalizer.models import CliDetectionResult
from charset_normalizer.version import __version__
def query_yes_no(question: str, default: str = "yes") -> bool: # Defensive:
"""Ask a yes/no question via input() and return the answer as a bool."""
prompt = " [Y/n] " if default == "yes" else " [y/N] "
while True:
choice = input(question + prompt).strip().lower()
if not choice:
return default == "yes"
if choice in ("y", "yes"):
return True
if choice in ("n", "no"):
return False
print("Please respond with 'y' or 'n'.")
class FileType:
"""Factory for creating file object types
Instances of FileType are typically passed as type= arguments to the
ArgumentParser add_argument() method.
Keyword Arguments:
- mode -- A string indicating how the file is to be opened. Accepts the
same values as the builtin open() function.
- bufsize -- The file's desired buffer size. Accepts the same values as
the builtin open() function.
- encoding -- The file's encoding. Accepts the same values as the
builtin open() function.
- errors -- A string indicating how encoding and decoding errors are to
be handled. Accepts the same value as the builtin open() function.
Backported from CPython 3.12
"""
def __init__(
self,
mode: str = "r",
bufsize: int = -1,
encoding: str | None = None,
errors: str | None = None,
):
self._mode = mode
self._bufsize = bufsize
self._encoding = encoding
self._errors = errors
def __call__(self, string: str) -> typing.IO: # type: ignore[type-arg]
# the special argument "-" means sys.std{in,out}
if string == "-":
if "r" in self._mode:
return sys.stdin.buffer if "b" in self._mode else sys.stdin
elif any(c in self._mode for c in "wax"):
return sys.stdout.buffer if "b" in self._mode else sys.stdout
else:
msg = f'argument "-" with mode {self._mode}'
raise ValueError(msg)
# all other arguments are used as file names
try:
return open(string, self._mode, self._bufsize, self._encoding, self._errors)
except OSError as e:
message = f"can't open '{string}': {e}"
raise argparse.ArgumentTypeError(message)
def __repr__(self) -> str:
args = self._mode, self._bufsize
kwargs = [("encoding", self._encoding), ("errors", self._errors)]
args_str = ", ".join(
[repr(arg) for arg in args if arg != -1]
+ [f"{kw}={arg!r}" for kw, arg in kwargs if arg is not None]
)
return f"{type(self).__name__}({args_str})"
def cli_detect(argv: list[str] | None = None) -> int:
"""
CLI assistant using ARGV and ArgumentParser
:param argv:
:return: 0 if everything is fine, anything else equal trouble
"""
parser = argparse.ArgumentParser(
description="The Real First Universal Charset Detector. "
"Discover originating encoding used on text file. "
"Normalize text to unicode."
)
parser.add_argument(
"files", type=FileType("rb"), nargs="+", help="File(s) to be analysed"
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
default=False,
dest="verbose",
help="Display complementary information about file if any. "
"Stdout will contain logs about the detection process.",
)
parser.add_argument(
"-a",
"--with-alternative",
action="store_true",
default=False,
dest="alternatives",
help="Output complementary possibilities if any. Top-level JSON WILL be a list.",
)
parser.add_argument(
"-n",
"--normalize",
action="store_true",
default=False,
dest="normalize",
help="Permit to normalize input file. If not set, program does not write anything.",
)
parser.add_argument(
"-m",
"--minimal",
action="store_true",
default=False,
dest="minimal",
help="Only output the charset detected to STDOUT. Disabling JSON output.",
)
parser.add_argument(
"-r",
"--replace",
action="store_true",
default=False,
dest="replace",
help="Replace file when trying to normalize it instead of creating a new one.",
)
parser.add_argument(
"-f",
"--force",
action="store_true",
default=False,
dest="force",
help="Replace file without asking if you are sure, use this flag with caution.",
)
parser.add_argument(
"-i",
"--no-preemptive",
action="store_true",
default=False,
dest="no_preemptive",
help="Disable looking at a charset declaration to hint the detector.",
)
parser.add_argument(
"-t",
"--threshold",
action="store",
default=0.2,
type=float,
dest="threshold",
help="Define a custom maximum amount of noise allowed in decoded content. 0. <= noise <= 1.",
)
parser.add_argument(
"--version",
action="version",
version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format(
__version__,
python_version(),
unidata_version,
"OFF" if md_module.__file__.lower().endswith(".py") else "ON",
),
help="Show version information and exit.",
)
args = parser.parse_args(argv)
if args.replace is True and args.normalize is False:
if args.files:
for my_file in args.files:
my_file.close()
print("Use --replace in addition of --normalize only.", file=sys.stderr)
return 1
if args.force is True and args.replace is False:
if args.files:
for my_file in args.files:
my_file.close()
print("Use --force in addition of --replace only.", file=sys.stderr)
return 1
if args.threshold < 0.0 or args.threshold > 1.0:
if args.files:
for my_file in args.files:
my_file.close()
print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr)
return 1
x_ = []
for my_file in args.files:
matches = from_fp(
my_file,
threshold=args.threshold,
explain=args.verbose,
preemptive_behaviour=args.no_preemptive is False,
)
best_guess = matches.best()
if best_guess is None:
print(
'Unable to identify originating encoding for "{}". {}'.format(
my_file.name,
(
"Maybe try increasing maximum amount of chaos."
if args.threshold < 1.0
else ""
),
),
file=sys.stderr,
)
x_.append(
CliDetectionResult(
abspath(my_file.name),
None,
[],
[],
"Unknown",
[],
False,
1.0,
0.0,
None,
True,
)
)
else:
cli_result = CliDetectionResult(
abspath(my_file.name),
best_guess.encoding,
best_guess.encoding_aliases,
[
cp
for cp in best_guess.could_be_from_charset
if cp != best_guess.encoding
],
best_guess.language,
best_guess.alphabets,
best_guess.bom,
best_guess.percent_chaos,
best_guess.percent_coherence,
None,
True,
)
x_.append(cli_result)
if len(matches) > 1 and args.alternatives:
for el in matches:
if el != best_guess:
x_.append(
CliDetectionResult(
abspath(my_file.name),
el.encoding,
el.encoding_aliases,
[
cp
for cp in el.could_be_from_charset
if cp != el.encoding
],
el.language,
el.alphabets,
el.bom,
el.percent_chaos,
el.percent_coherence,
None,
False,
)
)
if args.normalize is True:
if best_guess.encoding.startswith("utf") is True:
print(
'"{}" file does not need to be normalized, as it already came from unicode.'.format(
my_file.name
),
file=sys.stderr,
)
if my_file.closed is False:
my_file.close()
continue
dir_path = dirname(realpath(my_file.name))
file_name = basename(realpath(my_file.name))
o_: list[str] = file_name.split(".")
if args.replace is False:
o_.insert(-1, best_guess.encoding)
if my_file.closed is False:
my_file.close()
elif (
args.force is False
and query_yes_no(
'Are you sure to normalize "{}" by replacing it ?'.format(
my_file.name
),
"no",
)
is False
):
if my_file.closed is False:
my_file.close()
continue
try:
cli_result.unicode_path = join(dir_path, ".".join(o_))
with open(cli_result.unicode_path, "wb") as fp:
fp.write(best_guess.output())
except OSError as e: # Defensive:
print(str(e), file=sys.stderr)
if my_file.closed is False:
my_file.close()
return 2
if my_file.closed is False:
my_file.close()
if args.minimal is False:
from json import dumps
print(
dumps(
[el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__,
ensure_ascii=True,
indent=4,
)
)
else:
for my_file in args.files:
print(
", ".join(
[
el.encoding or "undefined"
for el in x_
if el.path == abspath(my_file.name)
]
)
)
return 0
if __name__ == "__main__": # Defensive:
cli_detect()
@@ -0,0 +1,79 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from warnings import warn
from .api import from_bytes
from .constant import CHARDET_CORRESPONDENCE, TOO_SMALL_SEQUENCE
if TYPE_CHECKING:
from typing import TypedDict
class ResultDict(TypedDict):
encoding: str | None
language: str
confidence: float | None
def detect(
byte_str: bytes, should_rename_legacy: bool = False, **kwargs: Any
) -> ResultDict:
"""
chardet legacy method
Detect the encoding of the given byte string. It should be mostly backward-compatible.
Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it)
This function is deprecated and should be used to migrate your project easily, consult the documentation for
further information. Not planned for removal.
:param byte_str: The byte sequence to examine.
:param should_rename_legacy: Should we rename legacy encodings
to their more modern equivalents?
"""
if len(kwargs):
warn(
f"charset-normalizer disregard arguments '{','.join(list(kwargs.keys()))}' in legacy function detect()"
)
if not isinstance(byte_str, (bytearray, bytes)):
raise TypeError( # pragma: nocover
f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
)
if isinstance(byte_str, bytearray):
byte_str = bytes(byte_str)
r = from_bytes(byte_str).best()
encoding = r.encoding if r is not None else None
language = r.language if r is not None and r.language != "Unknown" else ""
confidence = 1.0 - r.chaos if r is not None else None
# automatically lower confidence
# on small bytes samples.
# https://github.com/jawah/charset_normalizer/issues/391
if (
confidence is not None
and confidence >= 0.9
and encoding
not in {
"utf_8",
"ascii",
}
and not r.bom # type: ignore[union-attr]
and len(byte_str) < TOO_SMALL_SEQUENCE
):
confidence -= 0.2
# Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process
# but chardet does return 'utf-8-sig' and it is a valid codec name.
if r is not None and encoding == "utf_8" and r.bom:
encoding += "_sig"
if not should_rename_legacy and encoding in CHARDET_CORRESPONDENCE:
encoding = CHARDET_CORRESPONDENCE[encoding]
return {
"encoding": encoding,
"language": language,
"confidence": confidence,
}
@@ -0,0 +1,387 @@
from __future__ import annotations
from encodings.aliases import aliases
from re import sub
from typing import Any, Iterator, List, Tuple
from .constant import RE_POSSIBLE_ENCODING_INDICATION, TOO_BIG_SEQUENCE
from .utils import iana_name, is_multi_byte_encoding, unicode_range
class CharsetMatch:
def __init__(
self,
payload: bytes | bytearray,
guessed_encoding: str,
mean_mess_ratio: float,
has_sig_or_bom: bool,
languages: CoherenceMatches,
decoded_payload: str | None = None,
preemptive_declaration: str | None = None,
):
self._payload: bytes | bytearray = payload
self._encoding: str = guessed_encoding
self._mean_mess_ratio: float = mean_mess_ratio
self._languages: CoherenceMatches = languages
self._has_sig_or_bom: bool = has_sig_or_bom
self._unicode_ranges: list[str] | None = None
self._leaves: list[CharsetMatch] = []
self._mean_coherence_ratio: float = 0.0
self._output_payload: bytes | None = None
self._output_encoding: str | None = None
self._string: str | None = decoded_payload
self._preemptive_declaration: str | None = preemptive_declaration
def __eq__(self, other: object) -> bool:
if not isinstance(other, CharsetMatch):
if isinstance(other, str):
# Use non-strict iana_name so an operand that is not a known
# encoding alias compares unequal instead of raising, keeping
# __eq__ total as the data model requires (mirrors the lookup
# in CharsetMatches.__getitem__).
return iana_name(other, False) == self.encoding
return False
return self.encoding == other.encoding and self.fingerprint == other.fingerprint
def __lt__(self, other: object) -> bool:
"""
Implemented to make sorted available upon CharsetMatches items.
"""
if not isinstance(other, CharsetMatch):
raise ValueError
chaos_difference: float = abs(self.chaos - other.chaos)
coherence_difference: float = abs(self.coherence - other.coherence)
# Below 0.5% difference --> Use Coherence
if chaos_difference < 0.005 and coherence_difference > 0.02:
return self.coherence > other.coherence
elif chaos_difference < 0.005 and coherence_difference <= 0.02:
# When having a difficult decision, use the result that decoded as many multi-byte as possible.
# preserve RAM usage!
if len(self._payload) >= TOO_BIG_SEQUENCE:
return self.chaos < other.chaos
return self.multi_byte_usage > other.multi_byte_usage
return self.chaos < other.chaos
@property
def multi_byte_usage(self) -> float:
# Empty payloads are valid (from_bytes(b"") returns a match); avoid /0.
raw_len = len(self.raw)
if raw_len == 0:
return 0.0
return 1.0 - (len(str(self)) / raw_len)
def __str__(self) -> str:
# Lazy Str Loading
if self._string is None:
self._string = str(self._payload, self._encoding, "strict")
# UTF-7 BOM is encoded in modified Base64 whose byte boundary
# can overlap with the next character, so raw-byte stripping
# is unreliable. Strip the decoded BOM character instead.
if (
self._has_sig_or_bom
and self._encoding == "utf_7"
and self._string
and self._string[0] == "\ufeff"
):
self._string = self._string[1:]
return self._string
def __repr__(self) -> str:
return f"<CharsetMatch '{self.encoding}' fp({self.fingerprint})>"
def add_submatch(self, other: CharsetMatch) -> None:
if not isinstance(other, CharsetMatch) or other == self:
raise ValueError(
"Unable to add instance <{}> as a submatch of a CharsetMatch".format(
other.__class__
)
)
other._string = None # Unload RAM usage; dirty trick.
self._leaves.append(other)
@property
def encoding(self) -> str:
return self._encoding
@property
def encoding_aliases(self) -> list[str]:
"""
Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855.
"""
also_known_as: list[str] = []
for u, p in aliases.items():
if self.encoding == u:
also_known_as.append(p)
elif self.encoding == p:
also_known_as.append(u)
return also_known_as
@property
def bom(self) -> bool:
return self._has_sig_or_bom
@property
def byte_order_mark(self) -> bool:
return self._has_sig_or_bom
@property
def languages(self) -> list[str]:
"""
Return the complete list of possible languages found in decoded sequence.
Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'.
"""
return [e[0] for e in self._languages]
@property
def language(self) -> str:
"""
Most probable language found in decoded sequence. If none were detected or inferred, the property will return
"Unknown".
"""
if not self._languages:
# Trying to infer the language based on the given encoding
# Its either English or we should not pronounce ourselves in certain cases.
if "ascii" in self.could_be_from_charset:
return "English"
# doing it there to avoid circular import
from charset_normalizer.cd import encoding_languages, mb_encoding_languages
languages = (
mb_encoding_languages(self.encoding)
if is_multi_byte_encoding(self.encoding)
else encoding_languages(self.encoding)
)
if len(languages) == 0 or "Latin Based" in languages:
return "Unknown"
return languages[0]
return self._languages[0][0]
@property
def chaos(self) -> float:
return self._mean_mess_ratio
@property
def coherence(self) -> float:
if not self._languages:
return 0.0
return self._languages[0][1]
@property
def percent_chaos(self) -> float:
return round(self.chaos * 100, ndigits=3)
@property
def percent_coherence(self) -> float:
return round(self.coherence * 100, ndigits=3)
@property
def raw(self) -> bytes | bytearray:
"""
Original untouched bytes.
"""
return self._payload
@property
def submatch(self) -> list[CharsetMatch]:
return self._leaves
@property
def has_submatch(self) -> bool:
return len(self._leaves) > 0
@property
def alphabets(self) -> list[str]:
if self._unicode_ranges is not None:
return self._unicode_ranges
# list detected ranges
detected_ranges: list[str | None] = [unicode_range(char) for char in str(self)]
# filter and sort
self._unicode_ranges = sorted(list({r for r in detected_ranges if r}))
return self._unicode_ranges
@property
def could_be_from_charset(self) -> list[str]:
"""
The complete list of encoding that output the exact SAME str result and therefore could be the originating
encoding.
This list does include the encoding available in property 'encoding'.
"""
return [self._encoding] + [m.encoding for m in self._leaves]
def output(self, encoding: str = "utf_8") -> bytes:
"""
Method to get re-encoded bytes payload using given target encoding. Default to UTF-8.
Any errors will be simply ignored by the encoder NOT replaced.
"""
if self._output_encoding is None or self._output_encoding != encoding:
self._output_encoding = encoding
decoded_string = str(self)
if (
self._preemptive_declaration is not None
and self._preemptive_declaration.lower()
not in ["utf-8", "utf8", "utf_8"]
):
patched_header = sub(
RE_POSSIBLE_ENCODING_INDICATION,
lambda m: m.string[m.span()[0] : m.span()[1]].replace(
m.groups()[0],
iana_name(self._output_encoding).replace("_", "-"), # type: ignore[arg-type]
),
decoded_string[:8192],
count=1,
)
decoded_string = patched_header + decoded_string[8192:]
self._output_payload = decoded_string.encode(encoding, "replace")
return self._output_payload # type: ignore
@property
def fingerprint(self) -> int:
"""
Retrieve a hash fingerprint of the decoded payload, used for deduplication.
"""
return hash(str(self))
class CharsetMatches:
"""
Container with every CharsetMatch items ordered by default from most probable to the less one.
Act like a list(iterable) but does not implements all related methods.
"""
def __init__(self, results: list[CharsetMatch] | None = None):
self._results: list[CharsetMatch] = sorted(results) if results else []
self._is_sorted: bool = True
def _ensure_sorted(self) -> None:
if not self._is_sorted:
self._results.sort()
self._is_sorted = True
def __iter__(self) -> Iterator[CharsetMatch]:
self._ensure_sorted()
yield from self._results
def __getitem__(self, item: int | str) -> CharsetMatch:
"""
Retrieve a single item either by its position or encoding name (alias may be used here).
Raise KeyError upon invalid index or encoding not present in results.
"""
if isinstance(item, int):
self._ensure_sorted()
return self._results[item]
if isinstance(item, str):
item = iana_name(item, False)
for result in self._results:
if item in result.could_be_from_charset:
return result
raise KeyError
def __len__(self) -> int:
return len(self._results)
def __bool__(self) -> bool:
return len(self._results) > 0
def append(self, item: CharsetMatch) -> None:
"""
Insert a single match. Will be inserted accordingly to preserve sort.
Can be inserted as a submatch.
"""
if not isinstance(item, CharsetMatch):
raise ValueError(
"Cannot append instance '{}' to CharsetMatches".format(
str(item.__class__)
)
)
# We should disable the submatch factoring when the input file is too heavy (conserve RAM usage)
if len(item.raw) < TOO_BIG_SEQUENCE:
for match in self._results:
if match.fingerprint == item.fingerprint and match.chaos == item.chaos:
match.add_submatch(item)
return
self._results.append(item)
self._is_sorted = False
def best(self) -> CharsetMatch | None:
"""
Simply return the first match. Strict equivalent to matches[0].
"""
if not self._results:
return None
self._ensure_sorted()
return self._results[0]
def first(self) -> CharsetMatch | None:
"""
Redundant method, call the method best(). Kept for BC reasons.
"""
return self.best()
CoherenceMatch = Tuple[str, float]
CoherenceMatches = List[CoherenceMatch]
class CliDetectionResult:
def __init__(
self,
path: str,
encoding: str | None,
encoding_aliases: list[str],
alternative_encodings: list[str],
language: str,
alphabets: list[str],
has_sig_or_bom: bool,
chaos: float,
coherence: float,
unicode_path: str | None,
is_preferred: bool,
):
self.path: str = path
self.unicode_path: str | None = unicode_path
self.encoding: str | None = encoding
self.encoding_aliases: list[str] = encoding_aliases
self.alternative_encodings: list[str] = alternative_encodings
self.language: str = language
self.alphabets: list[str] = alphabets
self.has_sig_or_bom: bool = has_sig_or_bom
self.chaos: float = chaos
self.coherence: float = coherence
self.is_preferred: bool = is_preferred
@property
def __dict__(self) -> dict[str, Any]: # type: ignore
return {
"path": self.path,
"encoding": self.encoding,
"encoding_aliases": self.encoding_aliases,
"alternative_encodings": self.alternative_encodings,
"language": self.language,
"alphabets": self.alphabets,
"has_sig_or_bom": self.has_sig_or_bom,
"chaos": self.chaos,
"coherence": self.coherence,
"unicode_path": self.unicode_path,
"is_preferred": self.is_preferred,
}
def to_json(self) -> str:
from json import dumps
return dumps(self.__dict__, ensure_ascii=True, indent=4)
@@ -0,0 +1,452 @@
from __future__ import annotations
import importlib
import logging
import unicodedata
from bisect import bisect_right
from codecs import IncrementalDecoder
from functools import lru_cache
from typing import Generator
from .constant import (
ENCODING_MARKS,
IANA_SUPPORTED_SIMILAR,
RE_POSSIBLE_ENCODING_INDICATION,
UNICODE_RANGES_COMBINED,
_SECONDARY_RANGE_NAMES,
COMMON_CJK_CHARACTERS,
_LATIN,
_CJK,
_HANGUL,
_KATAKANA,
_HIRAGANA,
_HALFWIDTH_KATAKANA,
_THAI,
_ARABIC,
_ARABIC_ISOLATED_FORM,
_LIGATURE,
_SUPERSCRIPT,
_SENTENCE_OPEN_PUNCTUATION,
_ACCENT_KEYWORDS,
_ACCENTUATED,
_KNOWN_MB_DECODERS,
_KNOWN_MB_CLASSES,
_IANA_NAMES,
_MULTIBYTE_SEARCH_RADIUS,
)
def _character_flags(character: str) -> int:
"""Compute all name-based classification flags with a single unicodedata.name() call."""
try:
desc: str = unicodedata.name(character)
except ValueError:
return 0
flags: int = 0
if "LATIN" in desc:
flags |= _LATIN
if "CJK" in desc:
flags |= _CJK
if "HANGUL" in desc:
flags |= _HANGUL
if "KATAKANA" in desc:
flags |= _KATAKANA
if "HALFWIDTH" in desc:
flags |= _HALFWIDTH_KATAKANA
if "HIRAGANA" in desc:
flags |= _HIRAGANA
if "THAI" in desc:
flags |= _THAI
if "ARABIC" in desc:
flags |= _ARABIC
if "ISOLATED FORM" in desc:
flags |= _ARABIC_ISOLATED_FORM
if "LIGATURE" in desc or desc.endswith("LETTER AE"):
flags |= _LIGATURE
if "SUPERSCRIPT" in desc:
flags |= _SUPERSCRIPT
if desc in {"INVERTED QUESTION MARK", "INVERTED EXCLAMATION MARK"}:
flags |= _SENTENCE_OPEN_PUNCTUATION
for kw in _ACCENT_KEYWORDS:
if kw in desc:
flags |= _ACCENTUATED
break
return flags
def is_accentuated(character: str) -> bool:
return bool(_character_flags(character) & _ACCENTUATED)
def remove_accent(character: str) -> str:
decomposed: str = unicodedata.decomposition(character)
if not decomposed:
return character
codes: list[str] = decomposed.split(" ")
return chr(int(codes[0], 16))
# Pre-built sorted lookup table for O(log n) binary search in unicode_range().
# Each entry is (range_start, range_end_exclusive, range_name).
_UNICODE_RANGES_SORTED: list[tuple[int, int, str]] = sorted(
(ord_range.start, ord_range.stop, name)
for name, ord_range in UNICODE_RANGES_COMBINED.items()
)
_UNICODE_RANGE_STARTS: list[int] = [e[0] for e in _UNICODE_RANGES_SORTED]
def unicode_range(character: str) -> str | None:
"""
Retrieve the Unicode range official name from a single character.
"""
character_ord: int = ord(character)
if character_ord < 32:
return "Control character"
if character_ord < 128:
return "Basic Latin"
# Binary search: find the rightmost range whose start <= character_ord
idx = bisect_right(_UNICODE_RANGE_STARTS, character_ord) - 1
if idx >= 0:
start, stop, name = _UNICODE_RANGES_SORTED[idx]
if character_ord < stop:
return name
return None
def is_latin(character: str) -> bool:
return bool(_character_flags(character) & _LATIN)
def is_punctuation(character: str) -> bool:
character_category: str = unicodedata.category(character)
if "P" in character_category:
return True
character_range: str | None = unicode_range(character)
if character_range is None:
return False
return "Punctuation" in character_range
def is_symbol(character: str) -> bool:
character_category: str = unicodedata.category(character)
if "S" in character_category or "N" in character_category:
return True
character_range: str | None = unicode_range(character)
if character_range is None:
return False
return "Forms" in character_range and character_category != "Lo"
def is_emoticon(character: str) -> bool:
character_range: str | None = unicode_range(character)
if character_range is None:
return False
return "Emoticons" in character_range or "Pictographs" in character_range
def is_separator(character: str) -> bool:
if character.isspace() or character in {"|", "+", "<", ">"}:
return True
character_category: str = unicodedata.category(character)
return "Z" in character_category or character_category in {"Po", "Pd", "Pc"}
def is_case_variable(character: str) -> bool:
return character.islower() != character.isupper()
def is_cjk(character: str) -> bool:
return bool(_character_flags(character) & _CJK)
def is_hiragana(character: str) -> bool:
return bool(_character_flags(character) & _HIRAGANA)
def is_katakana(character: str) -> bool:
return bool(_character_flags(character) & _KATAKANA)
def is_hangul(character: str) -> bool:
return bool(_character_flags(character) & _HANGUL)
def is_thai(character: str) -> bool:
return bool(_character_flags(character) & _THAI)
def is_arabic(character: str) -> bool:
return bool(_character_flags(character) & _ARABIC)
def is_arabic_isolated_form(character: str) -> bool:
return bool(_character_flags(character) & _ARABIC_ISOLATED_FORM)
def is_cjk_uncommon(character: str) -> bool:
return character not in COMMON_CJK_CHARACTERS
def is_unicode_range_secondary(range_name: str) -> bool:
return range_name in _SECONDARY_RANGE_NAMES
def is_unprintable(character: str) -> bool:
return (
not character.isspace() # includes \n \t \r \v
and not character.isprintable()
and character != "\x1a" # Why? Its the ASCII substitute character.
and character != "\ufeff" # bug discovered in Python,
# Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space.
)
def any_specified_encoding(
sequence: bytes | bytearray, search_zone: int = 8192
) -> str | None:
"""
Extract using ASCII-only decoder any specified encoding in the first n-bytes.
"""
if not isinstance(sequence, (bytes, bytearray)):
raise TypeError
seq_len: int = len(sequence)
# Cheap literal pre-filter.
search_bytes = sequence[: min(seq_len, search_zone)]
lowered_bytes = search_bytes.lower()
if b"coding" not in lowered_bytes and b"charset" not in lowered_bytes:
return None
decoded_zone: str = search_bytes.decode("ascii", errors="ignore")
for match in RE_POSSIBLE_ENCODING_INDICATION.finditer(decoded_zone):
specified_encoding = match.group(1).lower().replace("-", "_")
encoding_iana = _IANA_NAMES.get(specified_encoding)
if encoding_iana is not None:
return encoding_iana
return None
@lru_cache(maxsize=None)
def is_multi_byte_encoding(name: str) -> bool:
"""
Verify is a specific encoding is a multi byte one based on it IANA name
"""
if name in _KNOWN_MB_DECODERS:
return True
# Besides the Unicode family above, every multibyte codec shipped with
# Python is implemented by _multibytecodec through exactly one of the six
# cjkcodecs providers below. Probing those providers directly (getcodec)
# classifies a name without importing its "encodings.<name>" module:
# classifying the whole IANA_SUPPORTED list would otherwise import many
# modules and dominate "import charset_normalizer" wall time.
# see https://github.com/jawah/charset_normalizer/issues/742
for provider in _KNOWN_MB_CLASSES:
try:
importlib.import_module(provider).getcodec(name) # type: ignore[attr-defined]
except (ImportError, AttributeError, LookupError): # Defensive: edge cases
continue
return True
return False
def identify_sig_or_bom(sequence: bytes | bytearray) -> tuple[str | None, bytes]:
"""
Identify and extract SIG/BOM in given sequence.
"""
for iana_encoding in ENCODING_MARKS:
marks: bytes | list[bytes] = ENCODING_MARKS[iana_encoding]
if isinstance(marks, bytes):
marks = [marks]
for mark in marks:
if sequence.startswith(mark):
return iana_encoding, mark
return None, b""
def should_strip_sig_or_bom(iana_encoding: str) -> bool:
return iana_encoding not in {"utf_16", "utf_32"}
def iana_name(cp_name: str, strict: bool = True) -> str:
"""Returns the Python normalized encoding name (Not the IANA official name)."""
cp_name = cp_name.lower().replace("-", "_")
encoding_iana = _IANA_NAMES.get(cp_name)
if encoding_iana is not None:
return encoding_iana
if strict:
raise ValueError(f"Unable to retrieve IANA for '{cp_name}'")
return cp_name
def cp_similarity(iana_name_a: str, iana_name_b: str) -> float:
if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b):
return 0.0
decoder_a = importlib.import_module(f"encodings.{iana_name_a}").IncrementalDecoder
decoder_b = importlib.import_module(f"encodings.{iana_name_b}").IncrementalDecoder
id_a: IncrementalDecoder = decoder_a(errors="ignore")
id_b: IncrementalDecoder = decoder_b(errors="ignore")
character_match_count: int = 0
for i in range(256):
to_be_decoded: bytes = bytes([i])
if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded):
character_match_count += 1
return character_match_count / 256
def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool:
"""
Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
the function cp_similarity.
"""
return (
iana_name_a in IANA_SUPPORTED_SIMILAR
and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a]
)
def set_logging_handler(
name: str = "charset_normalizer",
level: int = logging.INFO,
format_string: str = "%(asctime)s | %(levelname)s | %(message)s",
) -> None:
logger = logging.getLogger(name)
logger.setLevel(level)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(format_string))
logger.addHandler(handler)
def cut_sequence_chunks(
sequences: bytes | bytearray,
encoding_iana: str,
offsets: range,
chunk_size: int,
bom_or_sig_available: bool,
strip_sig_or_bom: bool,
sig_payload: bytes,
is_multi_byte_decoder: bool,
decoded_payload: str | None = None,
deferred_decoding: bool = False,
) -> Generator[str, None, None]:
# iso2022 codec is stateful, generic mb cuter isn't going to cut it!
if decoded_payload and encoding_iana.startswith("iso2022_"):
decoded_length = len(decoded_payload)
sequence_length = len(sequences)
for i in offsets:
decoded_offset = i * decoded_length // sequence_length
chunk = decoded_payload[decoded_offset : decoded_offset + chunk_size]
if not chunk:
break
yield chunk
elif decoded_payload and not is_multi_byte_decoder:
for i in offsets:
chunk = decoded_payload[i : i + chunk_size]
if not chunk:
break
yield chunk
elif deferred_decoding:
# Deferred single-byte probing: the whole payload is not decoded
# yet. Single-byte codecs are stateless (1 byte == 1 char), hence
# decode(base)[i:j] == decode(base[i:j]): slicing the raw bytes
# yields exactly the chunks the branch above would have produced,
# short trailing chunks included, and raises UnicodeDecodeError on
# invalid bytes just like the whole-payload decode would.
base_bytes = (
sequences if not strip_sig_or_bom else sequences[len(sig_payload) :]
)
for i in offsets:
cut_sequence = base_bytes[i : i + chunk_size]
if not cut_sequence:
break
yield str(cut_sequence, encoding_iana)
else:
for i in offsets:
chunk_end = i + chunk_size
if chunk_end > len(sequences) + 8:
continue
cut_sequence = sequences[i : i + chunk_size]
if bom_or_sig_available and not strip_sig_or_bom:
cut_sequence = sig_payload + cut_sequence
chunk = cut_sequence.decode(
encoding_iana,
errors="ignore" if is_multi_byte_decoder else "strict",
)
# multi-byte bad cutting detector and adjustment
# not the cleanest way to perform that fix but clever enough for now.
if is_multi_byte_decoder and i > 0:
chunk_partial_size_chk: int = min(chunk_size, 16)
chunk_prefix = chunk[:chunk_partial_size_chk]
found_nearby = False
if decoded_payload:
decoded_length = len(decoded_payload)
expected_offset = i * decoded_length // len(sequences)
search_start = max(0, expected_offset - _MULTIBYTE_SEARCH_RADIUS)
search_end = min(
decoded_length, expected_offset + _MULTIBYTE_SEARCH_RADIUS
)
found_nearby = (
decoded_payload.find(chunk_prefix, search_start, search_end)
>= 0
)
if (
decoded_payload
and not found_nearby
and chunk_prefix not in decoded_payload
):
for j in range(i, i - 4, -1):
cut_sequence = sequences[j:chunk_end]
if bom_or_sig_available and not strip_sig_or_bom:
cut_sequence = sig_payload + cut_sequence
chunk = cut_sequence.decode(encoding_iana, errors="ignore")
if chunk[:chunk_partial_size_chk] in decoded_payload:
break
yield chunk
@@ -0,0 +1,8 @@
"""
Expose version
"""
from __future__ import annotations
__version__ = "3.5.1"
VERSION = __version__.split(".")
@@ -0,0 +1,154 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defused xml.etree.ElementTree facade
"""
from __future__ import print_function, absolute_import
import sys
import warnings
from xml.etree.ElementTree import ParseError
from xml.etree.ElementTree import TreeBuilder as _TreeBuilder
from xml.etree.ElementTree import parse as _parse
from xml.etree.ElementTree import tostring
from .common import PY3
if PY3:
import importlib
else:
from xml.etree.ElementTree import XMLParser as _XMLParser
from xml.etree.ElementTree import iterparse as _iterparse
from .common import (
DTDForbidden,
EntitiesForbidden,
ExternalReferenceForbidden,
_generate_etree_functions,
)
__origin__ = "xml.etree.ElementTree"
def _get_py3_cls():
"""Python 3.3 hides the pure Python code but defusedxml requires it.
The code is based on test.support.import_fresh_module().
"""
pymodname = "xml.etree.ElementTree"
cmodname = "_elementtree"
pymod = sys.modules.pop(pymodname, None)
cmod = sys.modules.pop(cmodname, None)
sys.modules[cmodname] = None
try:
pure_pymod = importlib.import_module(pymodname)
finally:
# restore module
sys.modules[pymodname] = pymod
if cmod is not None:
sys.modules[cmodname] = cmod
else:
sys.modules.pop(cmodname, None)
# restore attribute on original package
etree_pkg = sys.modules["xml.etree"]
if pymod is not None:
etree_pkg.ElementTree = pymod
elif hasattr(etree_pkg, "ElementTree"):
del etree_pkg.ElementTree
_XMLParser = pure_pymod.XMLParser
_iterparse = pure_pymod.iterparse
# patch pure module to use ParseError from C extension
pure_pymod.ParseError = ParseError
return _XMLParser, _iterparse
if PY3:
_XMLParser, _iterparse = _get_py3_cls()
_sentinel = object()
class DefusedXMLParser(_XMLParser):
def __init__(
self,
html=_sentinel,
target=None,
encoding=None,
forbid_dtd=False,
forbid_entities=True,
forbid_external=True,
):
# Python 2.x old style class
_XMLParser.__init__(self, target=target, encoding=encoding)
if html is not _sentinel:
# the 'html' argument has been deprecated and ignored in all
# supported versions of Python. Python 3.8 finally removed it.
if html:
raise TypeError("'html=True' is no longer supported.")
else:
warnings.warn(
"'html' keyword argument is no longer supported. Pass "
"in arguments as keyword arguments.",
category=DeprecationWarning,
)
self.forbid_dtd = forbid_dtd
self.forbid_entities = forbid_entities
self.forbid_external = forbid_external
if PY3:
parser = self.parser
else:
parser = self._parser
if self.forbid_dtd:
parser.StartDoctypeDeclHandler = self.defused_start_doctype_decl
if self.forbid_entities:
parser.EntityDeclHandler = self.defused_entity_decl
parser.UnparsedEntityDeclHandler = self.defused_unparsed_entity_decl
if self.forbid_external:
parser.ExternalEntityRefHandler = self.defused_external_entity_ref_handler
def defused_start_doctype_decl(self, name, sysid, pubid, has_internal_subset):
raise DTDForbidden(name, sysid, pubid)
def defused_entity_decl(
self, name, is_parameter_entity, value, base, sysid, pubid, notation_name
):
raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name)
def defused_unparsed_entity_decl(self, name, base, sysid, pubid, notation_name):
# expat 1.2
raise EntitiesForbidden(name, None, base, sysid, pubid, notation_name) # pragma: no cover
def defused_external_entity_ref_handler(self, context, base, sysid, pubid):
raise ExternalReferenceForbidden(context, base, sysid, pubid)
# aliases
# XMLParse is a typo, keep it for backwards compatibility
XMLTreeBuilder = XMLParse = XMLParser = DefusedXMLParser
parse, iterparse, fromstring = _generate_etree_functions(
DefusedXMLParser, _TreeBuilder, _parse, _iterparse
)
XML = fromstring
__all__ = [
"ParseError",
"XML",
"XMLParse",
"XMLParser",
"XMLTreeBuilder",
"fromstring",
"iterparse",
"parse",
"tostring",
]
@@ -0,0 +1,67 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defuse XML bomb denial of service vulnerabilities
"""
from __future__ import print_function, absolute_import
import warnings
from .common import (
DefusedXmlException,
DTDForbidden,
EntitiesForbidden,
ExternalReferenceForbidden,
NotSupportedError,
_apply_defusing,
)
def defuse_stdlib():
"""Monkey patch and defuse all stdlib packages
:warning: The monkey patch is an EXPERIMETNAL feature.
"""
defused = {}
with warnings.catch_warnings():
from . import cElementTree
from . import ElementTree
from . import minidom
from . import pulldom
from . import sax
from . import expatbuilder
from . import expatreader
from . import xmlrpc
xmlrpc.monkey_patch()
defused[xmlrpc] = None
defused_mods = [
cElementTree,
ElementTree,
minidom,
pulldom,
sax,
expatbuilder,
expatreader,
]
for defused_mod in defused_mods:
stdlib_mod = _apply_defusing(defused_mod)
defused[defused_mod] = stdlib_mod
return defused
__version__ = "0.7.1"
__all__ = [
"DefusedXmlException",
"DTDForbidden",
"EntitiesForbidden",
"ExternalReferenceForbidden",
"NotSupportedError",
]
@@ -0,0 +1,62 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defused xml.etree.cElementTree
"""
from __future__ import absolute_import
import warnings
from .common import _generate_etree_functions
from xml.etree.cElementTree import TreeBuilder as _TreeBuilder
from xml.etree.cElementTree import parse as _parse
from xml.etree.cElementTree import tostring
# iterparse from ElementTree!
from xml.etree.ElementTree import iterparse as _iterparse
# This module is an alias for ElementTree just like xml.etree.cElementTree
from .ElementTree import (
XML,
XMLParse,
XMLParser,
XMLTreeBuilder,
fromstring,
iterparse,
parse,
tostring,
DefusedXMLParser,
ParseError,
)
__origin__ = "xml.etree.cElementTree"
warnings.warn(
"defusedxml.cElementTree is deprecated, import from defusedxml.ElementTree instead.",
category=DeprecationWarning,
stacklevel=2,
)
# XMLParse is a typo, keep it for backwards compatibility
XMLTreeBuilder = XMLParse = XMLParser = DefusedXMLParser
parse, iterparse, fromstring = _generate_etree_functions(
DefusedXMLParser, _TreeBuilder, _parse, _iterparse
)
XML = fromstring
__all__ = [
"ParseError",
"XML",
"XMLParse",
"XMLParser",
"XMLTreeBuilder",
"fromstring",
"iterparse",
"parse",
"tostring",
]
@@ -0,0 +1,129 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Common constants, exceptions and helpe functions
"""
import sys
import xml.parsers.expat
PY3 = sys.version_info[0] == 3
# Fail early when pyexpat is not installed correctly
if not hasattr(xml.parsers.expat, "ParserCreate"):
raise ImportError("pyexpat") # pragma: no cover
class DefusedXmlException(ValueError):
"""Base exception"""
def __repr__(self):
return str(self)
class DTDForbidden(DefusedXmlException):
"""Document type definition is forbidden"""
def __init__(self, name, sysid, pubid):
super(DTDForbidden, self).__init__()
self.name = name
self.sysid = sysid
self.pubid = pubid
def __str__(self):
tpl = "DTDForbidden(name='{}', system_id={!r}, public_id={!r})"
return tpl.format(self.name, self.sysid, self.pubid)
class EntitiesForbidden(DefusedXmlException):
"""Entity definition is forbidden"""
def __init__(self, name, value, base, sysid, pubid, notation_name):
super(EntitiesForbidden, self).__init__()
self.name = name
self.value = value
self.base = base
self.sysid = sysid
self.pubid = pubid
self.notation_name = notation_name
def __str__(self):
tpl = "EntitiesForbidden(name='{}', system_id={!r}, public_id={!r})"
return tpl.format(self.name, self.sysid, self.pubid)
class ExternalReferenceForbidden(DefusedXmlException):
"""Resolving an external reference is forbidden"""
def __init__(self, context, base, sysid, pubid):
super(ExternalReferenceForbidden, self).__init__()
self.context = context
self.base = base
self.sysid = sysid
self.pubid = pubid
def __str__(self):
tpl = "ExternalReferenceForbidden(system_id='{}', public_id={})"
return tpl.format(self.sysid, self.pubid)
class NotSupportedError(DefusedXmlException):
"""The operation is not supported"""
def _apply_defusing(defused_mod):
assert defused_mod is sys.modules[defused_mod.__name__]
stdlib_name = defused_mod.__origin__
__import__(stdlib_name, {}, {}, ["*"])
stdlib_mod = sys.modules[stdlib_name]
stdlib_names = set(dir(stdlib_mod))
for name, obj in vars(defused_mod).items():
if name.startswith("_") or name not in stdlib_names:
continue
setattr(stdlib_mod, name, obj)
return stdlib_mod
def _generate_etree_functions(DefusedXMLParser, _TreeBuilder, _parse, _iterparse):
"""Factory for functions needed by etree, dependent on whether
cElementTree or ElementTree is used."""
def parse(source, parser=None, forbid_dtd=False, forbid_entities=True, forbid_external=True):
if parser is None:
parser = DefusedXMLParser(
target=_TreeBuilder(),
forbid_dtd=forbid_dtd,
forbid_entities=forbid_entities,
forbid_external=forbid_external,
)
return _parse(source, parser)
def iterparse(
source,
events=None,
parser=None,
forbid_dtd=False,
forbid_entities=True,
forbid_external=True,
):
if parser is None:
parser = DefusedXMLParser(
target=_TreeBuilder(),
forbid_dtd=forbid_dtd,
forbid_entities=forbid_entities,
forbid_external=forbid_external,
)
return _iterparse(source, events, parser)
def fromstring(text, forbid_dtd=False, forbid_entities=True, forbid_external=True):
parser = DefusedXMLParser(
target=_TreeBuilder(),
forbid_dtd=forbid_dtd,
forbid_entities=forbid_entities,
forbid_external=forbid_external,
)
parser.feed(text)
return parser.close()
return parse, iterparse, fromstring
@@ -0,0 +1,107 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defused xml.dom.expatbuilder
"""
from __future__ import print_function, absolute_import
from xml.dom.expatbuilder import ExpatBuilder as _ExpatBuilder
from xml.dom.expatbuilder import Namespaces as _Namespaces
from .common import DTDForbidden, EntitiesForbidden, ExternalReferenceForbidden
__origin__ = "xml.dom.expatbuilder"
class DefusedExpatBuilder(_ExpatBuilder):
"""Defused document builder"""
def __init__(
self, options=None, forbid_dtd=False, forbid_entities=True, forbid_external=True
):
_ExpatBuilder.__init__(self, options)
self.forbid_dtd = forbid_dtd
self.forbid_entities = forbid_entities
self.forbid_external = forbid_external
def defused_start_doctype_decl(self, name, sysid, pubid, has_internal_subset):
raise DTDForbidden(name, sysid, pubid)
def defused_entity_decl(
self, name, is_parameter_entity, value, base, sysid, pubid, notation_name
):
raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name)
def defused_unparsed_entity_decl(self, name, base, sysid, pubid, notation_name):
# expat 1.2
raise EntitiesForbidden(name, None, base, sysid, pubid, notation_name) # pragma: no cover
def defused_external_entity_ref_handler(self, context, base, sysid, pubid):
raise ExternalReferenceForbidden(context, base, sysid, pubid)
def install(self, parser):
_ExpatBuilder.install(self, parser)
if self.forbid_dtd:
parser.StartDoctypeDeclHandler = self.defused_start_doctype_decl
if self.forbid_entities:
# if self._options.entities:
parser.EntityDeclHandler = self.defused_entity_decl
parser.UnparsedEntityDeclHandler = self.defused_unparsed_entity_decl
if self.forbid_external:
parser.ExternalEntityRefHandler = self.defused_external_entity_ref_handler
class DefusedExpatBuilderNS(_Namespaces, DefusedExpatBuilder):
"""Defused document builder that supports namespaces."""
def install(self, parser):
DefusedExpatBuilder.install(self, parser)
if self._options.namespace_declarations:
parser.StartNamespaceDeclHandler = self.start_namespace_decl_handler
def reset(self):
DefusedExpatBuilder.reset(self)
self._initNamespaces()
def parse(file, namespaces=True, forbid_dtd=False, forbid_entities=True, forbid_external=True):
"""Parse a document, returning the resulting Document node.
'file' may be either a file name or an open file object.
"""
if namespaces:
build_builder = DefusedExpatBuilderNS
else:
build_builder = DefusedExpatBuilder
builder = build_builder(
forbid_dtd=forbid_dtd, forbid_entities=forbid_entities, forbid_external=forbid_external
)
if isinstance(file, str):
fp = open(file, "rb")
try:
result = builder.parseFile(fp)
finally:
fp.close()
else:
result = builder.parseFile(file)
return result
def parseString(
string, namespaces=True, forbid_dtd=False, forbid_entities=True, forbid_external=True
):
"""Parse a document from a string, returning the resulting
Document node.
"""
if namespaces:
build_builder = DefusedExpatBuilderNS
else:
build_builder = DefusedExpatBuilder
builder = build_builder(
forbid_dtd=forbid_dtd, forbid_entities=forbid_entities, forbid_external=forbid_external
)
return builder.parseString(string)
@@ -0,0 +1,61 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defused xml.sax.expatreader
"""
from __future__ import print_function, absolute_import
from xml.sax.expatreader import ExpatParser as _ExpatParser
from .common import DTDForbidden, EntitiesForbidden, ExternalReferenceForbidden
__origin__ = "xml.sax.expatreader"
class DefusedExpatParser(_ExpatParser):
"""Defused SAX driver for the pyexpat C module."""
def __init__(
self,
namespaceHandling=0,
bufsize=2 ** 16 - 20,
forbid_dtd=False,
forbid_entities=True,
forbid_external=True,
):
_ExpatParser.__init__(self, namespaceHandling, bufsize)
self.forbid_dtd = forbid_dtd
self.forbid_entities = forbid_entities
self.forbid_external = forbid_external
def defused_start_doctype_decl(self, name, sysid, pubid, has_internal_subset):
raise DTDForbidden(name, sysid, pubid)
def defused_entity_decl(
self, name, is_parameter_entity, value, base, sysid, pubid, notation_name
):
raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name)
def defused_unparsed_entity_decl(self, name, base, sysid, pubid, notation_name):
# expat 1.2
raise EntitiesForbidden(name, None, base, sysid, pubid, notation_name) # pragma: no cover
def defused_external_entity_ref_handler(self, context, base, sysid, pubid):
raise ExternalReferenceForbidden(context, base, sysid, pubid)
def reset(self):
_ExpatParser.reset(self)
parser = self._parser
if self.forbid_dtd:
parser.StartDoctypeDeclHandler = self.defused_start_doctype_decl
if self.forbid_entities:
parser.EntityDeclHandler = self.defused_entity_decl
parser.UnparsedEntityDeclHandler = self.defused_unparsed_entity_decl
if self.forbid_external:
parser.ExternalEntityRefHandler = self.defused_external_entity_ref_handler
def create_parser(*args, **kwargs):
return DefusedExpatParser(*args, **kwargs)
@@ -0,0 +1,153 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""DEPRECATED Example code for lxml.etree protection
The code has NO protection against decompression bombs.
"""
from __future__ import print_function, absolute_import
import threading
import warnings
from lxml import etree as _etree
from .common import DTDForbidden, EntitiesForbidden, NotSupportedError
LXML3 = _etree.LXML_VERSION[0] >= 3
__origin__ = "lxml.etree"
tostring = _etree.tostring
warnings.warn(
"defusedxml.lxml is no longer supported and will be removed in a future release.",
category=DeprecationWarning,
stacklevel=2,
)
class RestrictedElement(_etree.ElementBase):
"""A restricted Element class that filters out instances of some classes"""
__slots__ = ()
# blacklist = (etree._Entity, etree._ProcessingInstruction, etree._Comment)
blacklist = _etree._Entity
def _filter(self, iterator):
blacklist = self.blacklist
for child in iterator:
if isinstance(child, blacklist):
continue
yield child
def __iter__(self):
iterator = super(RestrictedElement, self).__iter__()
return self._filter(iterator)
def iterchildren(self, tag=None, reversed=False):
iterator = super(RestrictedElement, self).iterchildren(tag=tag, reversed=reversed)
return self._filter(iterator)
def iter(self, tag=None, *tags):
iterator = super(RestrictedElement, self).iter(tag=tag, *tags)
return self._filter(iterator)
def iterdescendants(self, tag=None, *tags):
iterator = super(RestrictedElement, self).iterdescendants(tag=tag, *tags)
return self._filter(iterator)
def itersiblings(self, tag=None, preceding=False):
iterator = super(RestrictedElement, self).itersiblings(tag=tag, preceding=preceding)
return self._filter(iterator)
def getchildren(self):
iterator = super(RestrictedElement, self).__iter__()
return list(self._filter(iterator))
def getiterator(self, tag=None):
iterator = super(RestrictedElement, self).getiterator(tag)
return self._filter(iterator)
class GlobalParserTLS(threading.local):
"""Thread local context for custom parser instances"""
parser_config = {
"resolve_entities": False,
# 'remove_comments': True,
# 'remove_pis': True,
}
element_class = RestrictedElement
def createDefaultParser(self):
parser = _etree.XMLParser(**self.parser_config)
element_class = self.element_class
if self.element_class is not None:
lookup = _etree.ElementDefaultClassLookup(element=element_class)
parser.set_element_class_lookup(lookup)
return parser
def setDefaultParser(self, parser):
self._default_parser = parser
def getDefaultParser(self):
parser = getattr(self, "_default_parser", None)
if parser is None:
parser = self.createDefaultParser()
self.setDefaultParser(parser)
return parser
_parser_tls = GlobalParserTLS()
getDefaultParser = _parser_tls.getDefaultParser
def check_docinfo(elementtree, forbid_dtd=False, forbid_entities=True):
"""Check docinfo of an element tree for DTD and entity declarations
The check for entity declarations needs lxml 3 or newer. lxml 2.x does
not support dtd.iterentities().
"""
docinfo = elementtree.docinfo
if docinfo.doctype:
if forbid_dtd:
raise DTDForbidden(docinfo.doctype, docinfo.system_url, docinfo.public_id)
if forbid_entities and not LXML3:
# lxml < 3 has no iterentities()
raise NotSupportedError("Unable to check for entity declarations " "in lxml 2.x")
if forbid_entities:
for dtd in docinfo.internalDTD, docinfo.externalDTD:
if dtd is None:
continue
for entity in dtd.iterentities():
raise EntitiesForbidden(entity.name, entity.content, None, None, None, None)
def parse(source, parser=None, base_url=None, forbid_dtd=False, forbid_entities=True):
if parser is None:
parser = getDefaultParser()
elementtree = _etree.parse(source, parser, base_url=base_url)
check_docinfo(elementtree, forbid_dtd, forbid_entities)
return elementtree
def fromstring(text, parser=None, base_url=None, forbid_dtd=False, forbid_entities=True):
if parser is None:
parser = getDefaultParser()
rootelement = _etree.fromstring(text, parser, base_url=base_url)
elementtree = rootelement.getroottree()
check_docinfo(elementtree, forbid_dtd, forbid_entities)
return rootelement
XML = fromstring
def iterparse(*args, **kwargs):
raise NotSupportedError("defused lxml.etree.iterparse not available")
@@ -0,0 +1,63 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defused xml.dom.minidom
"""
from __future__ import print_function, absolute_import
from xml.dom.minidom import _do_pulldom_parse
from . import expatbuilder as _expatbuilder
from . import pulldom as _pulldom
__origin__ = "xml.dom.minidom"
def parse(
file, parser=None, bufsize=None, forbid_dtd=False, forbid_entities=True, forbid_external=True
):
"""Parse a file into a DOM by filename or file object."""
if parser is None and not bufsize:
return _expatbuilder.parse(
file,
forbid_dtd=forbid_dtd,
forbid_entities=forbid_entities,
forbid_external=forbid_external,
)
else:
return _do_pulldom_parse(
_pulldom.parse,
(file,),
{
"parser": parser,
"bufsize": bufsize,
"forbid_dtd": forbid_dtd,
"forbid_entities": forbid_entities,
"forbid_external": forbid_external,
},
)
def parseString(
string, parser=None, forbid_dtd=False, forbid_entities=True, forbid_external=True
):
"""Parse a file into a DOM from a string."""
if parser is None:
return _expatbuilder.parseString(
string,
forbid_dtd=forbid_dtd,
forbid_entities=forbid_entities,
forbid_external=forbid_external,
)
else:
return _do_pulldom_parse(
_pulldom.parseString,
(string,),
{
"parser": parser,
"forbid_dtd": forbid_dtd,
"forbid_entities": forbid_entities,
"forbid_external": forbid_external,
},
)
@@ -0,0 +1,41 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defused xml.dom.pulldom
"""
from __future__ import print_function, absolute_import
from xml.dom.pulldom import parse as _parse
from xml.dom.pulldom import parseString as _parseString
from .sax import make_parser
__origin__ = "xml.dom.pulldom"
def parse(
stream_or_string,
parser=None,
bufsize=None,
forbid_dtd=False,
forbid_entities=True,
forbid_external=True,
):
if parser is None:
parser = make_parser()
parser.forbid_dtd = forbid_dtd
parser.forbid_entities = forbid_entities
parser.forbid_external = forbid_external
return _parse(stream_or_string, parser, bufsize)
def parseString(
string, parser=None, forbid_dtd=False, forbid_entities=True, forbid_external=True
):
if parser is None:
parser = make_parser()
parser.forbid_dtd = forbid_dtd
parser.forbid_entities = forbid_entities
parser.forbid_external = forbid_external
return _parseString(string, parser)
@@ -0,0 +1,60 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defused xml.sax
"""
from __future__ import print_function, absolute_import
from xml.sax import InputSource as _InputSource
from xml.sax import ErrorHandler as _ErrorHandler
from . import expatreader
__origin__ = "xml.sax"
def parse(
source,
handler,
errorHandler=_ErrorHandler(),
forbid_dtd=False,
forbid_entities=True,
forbid_external=True,
):
parser = make_parser()
parser.setContentHandler(handler)
parser.setErrorHandler(errorHandler)
parser.forbid_dtd = forbid_dtd
parser.forbid_entities = forbid_entities
parser.forbid_external = forbid_external
parser.parse(source)
def parseString(
string,
handler,
errorHandler=_ErrorHandler(),
forbid_dtd=False,
forbid_entities=True,
forbid_external=True,
):
from io import BytesIO
if errorHandler is None:
errorHandler = _ErrorHandler()
parser = make_parser()
parser.setContentHandler(handler)
parser.setErrorHandler(errorHandler)
parser.forbid_dtd = forbid_dtd
parser.forbid_entities = forbid_entities
parser.forbid_external = forbid_external
inpsrc = _InputSource()
inpsrc.setByteStream(BytesIO(string))
parser.parse(inpsrc)
def make_parser(parser_list=[]):
return expatreader.create_parser()
@@ -0,0 +1,153 @@
# defusedxml
#
# Copyright (c) 2013 by Christian Heimes <[email protected]>
# Licensed to PSF under a Contributor Agreement.
# See https://www.python.org/psf/license for licensing details.
"""Defused xmlrpclib
Also defuses gzip bomb
"""
from __future__ import print_function, absolute_import
import io
from .common import DTDForbidden, EntitiesForbidden, ExternalReferenceForbidden, PY3
if PY3:
__origin__ = "xmlrpc.client"
from xmlrpc.client import ExpatParser
from xmlrpc import client as xmlrpc_client
from xmlrpc import server as xmlrpc_server
from xmlrpc.client import gzip_decode as _orig_gzip_decode
from xmlrpc.client import GzipDecodedResponse as _OrigGzipDecodedResponse
else:
__origin__ = "xmlrpclib"
from xmlrpclib import ExpatParser
import xmlrpclib as xmlrpc_client
xmlrpc_server = None
from xmlrpclib import gzip_decode as _orig_gzip_decode
from xmlrpclib import GzipDecodedResponse as _OrigGzipDecodedResponse
try:
import gzip
except ImportError: # pragma: no cover
gzip = None
# Limit maximum request size to prevent resource exhaustion DoS
# Also used to limit maximum amount of gzip decoded data in order to prevent
# decompression bombs
# A value of -1 or smaller disables the limit
MAX_DATA = 30 * 1024 * 1024 # 30 MB
def defused_gzip_decode(data, limit=None):
"""gzip encoded data -> unencoded data
Decode data using the gzip content encoding as described in RFC 1952
"""
if not gzip: # pragma: no cover
raise NotImplementedError
if limit is None:
limit = MAX_DATA
f = io.BytesIO(data)
gzf = gzip.GzipFile(mode="rb", fileobj=f)
try:
if limit < 0: # no limit
decoded = gzf.read()
else:
decoded = gzf.read(limit + 1)
except IOError: # pragma: no cover
raise ValueError("invalid data")
f.close()
gzf.close()
if limit >= 0 and len(decoded) > limit:
raise ValueError("max gzipped payload length exceeded")
return decoded
class DefusedGzipDecodedResponse(gzip.GzipFile if gzip else object):
"""a file-like object to decode a response encoded with the gzip
method, as described in RFC 1952.
"""
def __init__(self, response, limit=None):
# response doesn't support tell() and read(), required by
# GzipFile
if not gzip: # pragma: no cover
raise NotImplementedError
self.limit = limit = limit if limit is not None else MAX_DATA
if limit < 0: # no limit
data = response.read()
self.readlength = None
else:
data = response.read(limit + 1)
self.readlength = 0
if limit >= 0 and len(data) > limit:
raise ValueError("max payload length exceeded")
self.stringio = io.BytesIO(data)
gzip.GzipFile.__init__(self, mode="rb", fileobj=self.stringio)
def read(self, n):
if self.limit >= 0:
left = self.limit - self.readlength
n = min(n, left + 1)
data = gzip.GzipFile.read(self, n)
self.readlength += len(data)
if self.readlength > self.limit:
raise ValueError("max payload length exceeded")
return data
else:
return gzip.GzipFile.read(self, n)
def close(self):
gzip.GzipFile.close(self)
self.stringio.close()
class DefusedExpatParser(ExpatParser):
def __init__(self, target, forbid_dtd=False, forbid_entities=True, forbid_external=True):
ExpatParser.__init__(self, target)
self.forbid_dtd = forbid_dtd
self.forbid_entities = forbid_entities
self.forbid_external = forbid_external
parser = self._parser
if self.forbid_dtd:
parser.StartDoctypeDeclHandler = self.defused_start_doctype_decl
if self.forbid_entities:
parser.EntityDeclHandler = self.defused_entity_decl
parser.UnparsedEntityDeclHandler = self.defused_unparsed_entity_decl
if self.forbid_external:
parser.ExternalEntityRefHandler = self.defused_external_entity_ref_handler
def defused_start_doctype_decl(self, name, sysid, pubid, has_internal_subset):
raise DTDForbidden(name, sysid, pubid)
def defused_entity_decl(
self, name, is_parameter_entity, value, base, sysid, pubid, notation_name
):
raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name)
def defused_unparsed_entity_decl(self, name, base, sysid, pubid, notation_name):
# expat 1.2
raise EntitiesForbidden(name, None, base, sysid, pubid, notation_name) # pragma: no cover
def defused_external_entity_ref_handler(self, context, base, sysid, pubid):
raise ExternalReferenceForbidden(context, base, sysid, pubid)
def monkey_patch():
xmlrpc_client.FastParser = DefusedExpatParser
xmlrpc_client.GzipDecodedResponse = DefusedGzipDecodedResponse
xmlrpc_client.gzip_decode = defused_gzip_decode
if xmlrpc_server:
xmlrpc_server.gzip_decode = defused_gzip_decode
def unmonkey_patch():
xmlrpc_client.FastParser = None
xmlrpc_client.GzipDecodedResponse = _OrigGzipDecodedResponse
xmlrpc_client.gzip_decode = _orig_gzip_decode
if xmlrpc_server:
xmlrpc_server.gzip_decode = _orig_gzip_decode
@@ -0,0 +1,47 @@
from .core import (
IDNABidiError,
IDNAError,
InvalidCodepoint,
InvalidCodepointContext,
alabel,
check_bidi,
check_hyphen_ok,
check_initial_combiner,
check_label,
check_nfc,
decode,
encode,
ulabel,
uts46_remap,
valid_contextj,
valid_contexto,
valid_label_length,
valid_string_length,
)
from .idnadata import __version__ as unicode_version
from .intranges import intranges_contain
from .package_data import __version__
__all__ = [
"__version__",
"unicode_version",
"IDNABidiError",
"IDNAError",
"InvalidCodepoint",
"InvalidCodepointContext",
"alabel",
"check_bidi",
"check_hyphen_ok",
"check_initial_combiner",
"check_label",
"check_nfc",
"decode",
"encode",
"intranges_contain",
"ulabel",
"uts46_remap",
"valid_contextj",
"valid_contexto",
"valid_label_length",
"valid_string_length",
]
@@ -0,0 +1,6 @@
import sys
from .cli import main
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,132 @@
"""Command-line interface for the :mod:`idna` package.
Invoked via ``python -m idna``. See :func:`main` for the entry point.
"""
from __future__ import annotations
import argparse
import sys
from itertools import chain
from typing import IO, TYPE_CHECKING
from . import IDNAError, decode, encode, unicode_version
from .core import _alabel_prefix, _unicode_dots_re
from .package_data import __version__
if TYPE_CHECKING:
from collections.abc import Iterable
def _looks_like_alabel(s: str) -> bool:
"""Return True if any label in ``s`` carries the ``xn--`` ACE prefix."""
prefix = _alabel_prefix.decode("ascii")
return any(label.lower().startswith(prefix) for label in _unicode_dots_re.split(s))
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="python -m idna",
description=(
"Convert a domain name between its Unicode (U-label) and "
"ASCII-compatible (A-label) forms. With no mode flag, the "
"direction is chosen from the first input — if it contains "
"an xn-- label the stream is decoded, otherwise it is "
"encoded — and the same mode is applied to every remaining "
"input. UTS #46 mapping is applied by default; pass "
"--strict to disable it. When no domains are given on the "
"command line and stdin is piped, one domain per line is "
"read from stdin."
),
)
mode = parser.add_mutually_exclusive_group()
mode.add_argument(
"-e",
"--encode",
dest="mode",
action="store_const",
const="encode",
help="Encode the input to its ASCII A-label form.",
)
mode.add_argument(
"-d",
"--decode",
dest="mode",
action="store_const",
const="decode",
help="Decode the input from its ASCII A-label form.",
)
parser.add_argument(
"--strict",
action="store_true",
help="Disable the default UTS #46 mapping and apply IDNA 2008 rules verbatim.",
)
parser.add_argument(
"--version",
action="version",
version=f"idna {__version__} (Unicode {unicode_version})",
)
parser.add_argument(
"domain",
nargs="*",
help="One or more domain names to convert. Omit to read from stdin.",
)
return parser
def _iter_stdin(stream: IO[str]) -> Iterable[str]:
"""Yield non-empty stripped lines from ``stream``, ignoring blanks."""
for line in stream:
stripped = line.strip()
if stripped:
yield stripped
def _convert_one(domain: str, mode: str, uts46: bool) -> bool:
"""Convert ``domain`` and write the result; return ``False`` on failure."""
try:
if mode == "decode":
print(decode(domain, uts46=uts46))
else:
print(encode(domain, uts46=uts46).decode("ascii"))
except IDNAError as err:
print(f"idna: {mode} failed for {domain!r}: {err}", file=sys.stderr)
return False
return True
def main(argv: list[str] | None = None) -> int:
"""Entry point for ``python -m idna``.
When more than one domain is supplied (via positional arguments or
piped stdin) and no mode flag is given, the first input determines
the direction and that mode is applied uniformly to the rest.
:param argv: Argument list excluding the program name. Defaults to
:data:`sys.argv` when ``None``.
:returns: ``0`` on success, ``1`` if any conversion fails.
"""
parser = _build_parser()
args = parser.parse_args(argv)
uts46 = not args.strict
if args.domain:
domains: Iterable[str] = args.domain
elif not sys.stdin.isatty():
domains = _iter_stdin(sys.stdin)
else:
parser.error("a domain argument is required when stdin is a terminal")
iterator = iter(domains)
first = next(iterator, None)
if first is None:
return 0
mode = args.mode or ("decode" if _looks_like_alabel(first) else "encode")
results = [_convert_one(domain, mode, uts46) for domain in chain([first], iterator)]
return 0 if all(results) else 1
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,163 @@
from __future__ import annotations
import codecs
from typing import Any
from .core import IDNAError, _unicode_dots_re, alabel, decode, encode, ulabel
class Codec(codecs.Codec):
"""Stateless IDNA 2008 codec.
Implements the :class:`codecs.Codec` protocol so that the whole-domain
encoder (:func:`idna.encode`) and decoder (:func:`idna.decode`) are
accessible through the standard codec machinery as ``"idna2008"``.
Only the ``"strict"`` error handler is supported; any other handler
raises :exc:`~idna.IDNAError`.
"""
def encode(self, data: str, errors: str = "strict") -> tuple[bytes, int]: # ty: ignore[invalid-method-override]
if errors != "strict":
raise IDNAError(f'Unsupported error handling "{errors}"', code="unsupported_errors")
if not data:
return b"", 0
return encode(data), len(data)
def decode(self, data: bytes, errors: str = "strict") -> tuple[str, int]: # ty: ignore[invalid-method-override]
if errors != "strict":
raise IDNAError(f'Unsupported error handling "{errors}"', code="unsupported_errors")
if not data:
return "", 0
return decode(data), len(data)
class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
"""Incremental IDNA 2008 encoder.
Buffers a partial trailing label across calls until either the next
label separator is seen or ``final=True``, so that streamed input is
encoded one whole label at a time. Any of the four Unicode label
separators (``U+002E``, ``U+3002``, ``U+FF0E``, ``U+FF61``) ends a
label; the result always uses ``U+002E`` as the separator.
Only the ``"strict"`` error handler is supported.
"""
def _buffer_encode(self, data: str, errors: str, final: bool) -> tuple[bytes, int]: # ty: ignore[invalid-method-override]
if errors != "strict":
raise IDNAError(f'Unsupported error handling "{errors}"', code="unsupported_errors")
if not data:
return b"", 0
labels = _unicode_dots_re.split(data)
trailing_dot = b""
if labels:
if not labels[-1]:
trailing_dot = b"."
del labels[-1]
elif not final:
# Keep potentially unfinished label until the next call
del labels[-1]
if labels:
trailing_dot = b"."
result = []
size = 0
for label in labels:
result.append(alabel(label))
if size:
size += 1
size += len(label)
result_bytes = b".".join(result) + trailing_dot
size += len(trailing_dot)
return result_bytes, size
class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
"""Incremental IDNA 2008 decoder.
Buffers a partial trailing label across calls until either the next
label separator is seen or ``final=True``, so that streamed input is
decoded one whole label at a time.
Only the ``"strict"`` error handler is supported.
"""
def _buffer_decode(self, data: Any, errors: str, final: bool) -> tuple[str, int]: # ty: ignore[invalid-method-override]
if errors != "strict":
raise IDNAError(f'Unsupported error handling "{errors}"', code="unsupported_errors")
if not data:
return ("", 0)
if not isinstance(data, str):
try:
data = str(data, "ascii")
except UnicodeDecodeError as err:
raise IDNAError("Invalid ASCII in A-label", code="invalid_ascii") from err
labels = _unicode_dots_re.split(data)
trailing_dot = ""
if labels:
if not labels[-1]:
trailing_dot = "."
del labels[-1]
elif not final:
# Keep potentially unfinished label until the next call
del labels[-1]
if labels:
trailing_dot = "."
result = []
size = 0
for label in labels:
result.append(ulabel(label))
if size:
size += 1
size += len(label)
result_str = ".".join(result) + trailing_dot
size += len(trailing_dot)
return (result_str, size)
class StreamWriter(Codec, codecs.StreamWriter):
pass
class StreamReader(Codec, codecs.StreamReader):
pass
def search_function(name: str) -> codecs.CodecInfo | None:
"""Codec search function registered with :mod:`codecs`.
Returns a :class:`codecs.CodecInfo` for the ``"idna2008"`` codec name
so that ``str.encode("idna2008")`` and ``bytes.decode("idna2008")``
invoke the IDNA 2008 codec defined in this module.
:param name: The codec name being looked up.
:returns: A :class:`codecs.CodecInfo` instance if ``name`` is
``"idna2008"``, otherwise ``None``.
"""
if name != "idna2008":
return None
return codecs.CodecInfo(
name=name,
encode=Codec().encode,
decode=Codec().decode, # type: ignore
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamwriter=StreamWriter,
streamreader=StreamReader,
)
codecs.register(search_function)
@@ -0,0 +1,43 @@
from __future__ import annotations
from typing import Any
from .core import decode, encode
def ToASCII(label: str) -> bytes:
"""Compatibility shim for :rfc:`3490` ``ToASCII``.
Delegates to :func:`idna.encode` (IDNA 2008). Provided to ease porting
of code written against the legacy :mod:`encodings.idna` API; new code
should call :func:`idna.encode` directly.
:param label: The label or domain to encode.
:returns: The encoded form as ASCII :class:`bytes`.
"""
return encode(label)
def ToUnicode(label: bytes | bytearray) -> str:
"""Compatibility shim for :rfc:`3490` ``ToUnicode``.
Delegates to :func:`idna.decode` (IDNA 2008). Provided to ease porting
of code written against the legacy :mod:`encodings.idna` API; new code
should call :func:`idna.decode` directly.
:param label: The label or domain to decode.
:returns: The decoded Unicode form.
"""
return decode(label)
def nameprep(s: Any) -> None:
"""Stub for :rfc:`3491` Nameprep, which is not used by IDNA 2008.
IDNA 2008 (:rfc:`5891`) replaces Nameprep with the per-codepoint
validity classes from :rfc:`5892`; this function exists only to
return a clear error if legacy code attempts to call it.
:raises NotImplementedError: Always.
"""
raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol")
@@ -0,0 +1,862 @@
from __future__ import annotations
import bisect
import re
import unicodedata
import warnings
from typing import Literal
from . import idnadata
from .intranges import intranges_contain
_virama_combining_class = 9
_alabel_prefix = b"xn--"
_max_input_length = 1024
_STATUS_VALID, _STATUS_MAPPED, _STATUS_DEVIATION, _STATUS_IGNORED = b"VMDI"
_unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]")
_std3_disallowed_re = re.compile("[\x00-\x2c\x2f\x3a-\x40A-Z\x5b-\x60\x7b-\x7f]")
_bidi_rtl_first = frozenset({"R", "AL"})
_bidi_rtl_categories = frozenset({"R", "AL", "AN"})
_bidi_rtl_allowed = frozenset({"R", "AL", "AN", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"})
_bidi_rtl_valid_ending = frozenset({"R", "AL", "EN", "AN"})
_bidi_rtl_numeric = frozenset({"AN", "EN"})
_bidi_ltr_allowed = frozenset({"L", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"})
_bidi_ltr_valid_ending = frozenset({"L", "EN"})
_bidi_joiner_l_or_d = frozenset({"L", "D"})
_bidi_joiner_r_or_d = frozenset({"R", "D"})
def _joining_type(cp: int) -> str | None:
for jt, ranges in idnadata.joining_types.items():
if intranges_contain(cp, ranges):
return jt
return None
# Machine-readable identifiers for the rule an :class:`IDNAError` reports.
# These strings are stable and documented; exception message wording is not.
_ErrorCode = Literal[
"input_too_long",
"label_too_long",
"domain_too_long",
"empty_label",
"empty_domain",
"not_nfc",
"hyphen_3_4",
"hyphen_start_end",
"leading_combiner",
"disallowed_codepoint",
"contextj",
"contexto",
"unknown_codepoint",
"bidi_rule_1",
"bidi_rule_2",
"bidi_rule_3",
"bidi_rule_4",
"bidi_rule_5",
"bidi_rule_6",
"bidi_unknown_direction",
"invalid_alabel",
"non_canonical_alabel",
"invalid_ascii",
"invalid_utf8",
"uts46_disallowed",
"uts46_std3",
"unsupported_errors",
]
class IDNAError(UnicodeError):
"""Base exception for all IDNA-encoding related problems.
``str(err)`` is a human-readable description of the failure. The
exception also carries machine-readable attributes so callers do not
need to parse the message:
* ``code`` -- a short, stable identifier for the rule that failed, such
as ``"disallowed_codepoint"`` or ``"bidi_rule_2"``; the full list is
documented in the README. Message wording, by contrast, may change
between releases.
* ``text`` -- the label (or, for UTS #46 processing, the domain) that
was being validated;
* ``codepoint`` -- the offending codepoint, as an ``int``;
* ``position`` -- the 1-based index of the offending character within
``text``, matching the position quoted in the message.
Each is ``None`` when it does not apply.
"""
code: str | None
text: str | None
codepoint: int | None
position: int | None
def __init__(
self,
*args: object,
code: _ErrorCode | None = None,
text: str | None = None,
codepoint: int | None = None,
position: int | None = None,
) -> None:
super().__init__(*args)
self.code = code
self.text = text
self.codepoint = codepoint
self.position = position
class IDNABidiError(IDNAError):
"""Exception when bidirectional requirements are not satisfied"""
class InvalidCodepoint(IDNAError):
"""Exception when a disallowed or unallocated codepoint is used"""
class InvalidCodepointContext(IDNAError):
"""Exception when the codepoint is not valid in the context it is used"""
def _combining_class(cp: int) -> int:
v = unicodedata.combining(chr(cp))
if v == 0 and not unicodedata.name(chr(cp)):
raise ValueError("Unknown character in unicodedata")
return v
def _is_script(cp: str, script: str) -> bool:
return intranges_contain(ord(cp), idnadata.scripts[script])
def _punycode(s: str) -> bytes:
return s.encode("punycode")
def _unot(s: int) -> str:
return f"U+{s:04X}"
def valid_label_length(label: bytes | str) -> bool:
"""Check that a label does not exceed the maximum permitted length.
Per :rfc:`1035` (and :rfc:`5891` §4.2.4) a DNS label must not exceed
63 octets. The argument may be either a :class:`str` (a U-label, where
length is measured in characters) or :class:`bytes` (an A-label, where
length is measured in octets).
:param label: The label to check.
:returns: ``True`` if the label is within the length limit, otherwise
``False``.
"""
return len(label) <= 63
def valid_string_length(domain: bytes | str, trailing_dot: bool) -> bool:
"""Check that a full domain name does not exceed the maximum length.
Per :rfc:`1035`, a domain name is limited to 253 octets when no trailing
dot is present, or 254 octets when one is included.
:param domain: The full (possibly multi-label) domain name.
:param trailing_dot: ``True`` if ``domain`` includes a trailing ``.``.
:returns: ``True`` if the domain is within the length limit, otherwise
``False``.
"""
return len(domain) <= (254 if trailing_dot else 253)
def check_bidi(label: str, check_ltr: bool = False) -> bool:
"""Validate the Bidi Rule from :rfc:`5893` for a single label.
The Bidi Rule constrains how bidirectional characters (Hebrew, Arabic,
etc.) may appear within a label. By default the check is only applied
when the label contains at least one right-to-left character (Unicode
bidirectional categories ``R``, ``AL``, or ``AN``); set ``check_ltr``
to ``True`` to apply it to LTR-only labels as well.
:param label: The label to validate, as a Unicode string.
:param check_ltr: If ``True``, apply the rules even when the label
contains no RTL characters.
:returns: ``True`` if the label satisfies the Bidi Rule.
:raises IDNABidiError: If any of Bidi Rule conditions 1-6 are violated,
or if the directional category of a codepoint cannot be determined.
"""
if len(label) > _max_input_length:
raise IDNAError("Label too long", code="input_too_long")
# Bidi rules should only be applied if string contains RTL characters
bidi_label = False
for idx, cp in enumerate(label, 1):
direction = unicodedata.bidirectional(cp)
if direction == "":
# String likely comes from a newer version of Unicode
raise IDNABidiError(
f"Unknown directionality in label {label!r} at position {idx}",
code="bidi_unknown_direction",
text=label,
codepoint=ord(cp),
position=idx,
)
if direction in _bidi_rtl_categories:
bidi_label = True
if not bidi_label and not check_ltr:
return True
# Bidi rule 1
direction = unicodedata.bidirectional(label[0])
if direction in _bidi_rtl_first:
rtl = True
elif direction == "L":
rtl = False
else:
raise IDNABidiError(
f"First codepoint in label {label!r} must be directionality L, R or AL",
code="bidi_rule_1",
text=label,
codepoint=ord(label[0]),
position=1,
)
valid_ending = False
ending_idx = 1
number_type: str | None = None
for idx, cp in enumerate(label, 1):
direction = unicodedata.bidirectional(cp)
if rtl:
# Bidi rule 2
if direction not in _bidi_rtl_allowed:
raise IDNABidiError(
f"Invalid direction for codepoint at position {idx} in a right-to-left label",
code="bidi_rule_2",
text=label,
codepoint=ord(cp),
position=idx,
)
# Bidi rule 3
if direction in _bidi_rtl_valid_ending:
valid_ending = True
ending_idx = idx
elif direction != "NSM":
valid_ending = False
ending_idx = idx
# Bidi rule 4
if direction in _bidi_rtl_numeric:
if not number_type:
number_type = direction
elif number_type != direction:
raise IDNABidiError(
"Can not mix numeral types in a right-to-left label",
code="bidi_rule_4",
text=label,
codepoint=ord(cp),
position=idx,
)
else:
# Bidi rule 5
if direction not in _bidi_ltr_allowed:
raise IDNABidiError(
f"Invalid direction for codepoint at position {idx} in a left-to-right label",
code="bidi_rule_5",
text=label,
codepoint=ord(cp),
position=idx,
)
# Bidi rule 6
if direction in _bidi_ltr_valid_ending:
valid_ending = True
ending_idx = idx
elif direction != "NSM":
valid_ending = False
ending_idx = idx
if not valid_ending:
# Rules 3 and 6 concern the last character that is not a
# non-spacing mark, which is what ``ending_idx`` tracks.
raise IDNABidiError(
"Label ends with illegal codepoint directionality",
code="bidi_rule_3" if rtl else "bidi_rule_6",
text=label,
codepoint=ord(label[ending_idx - 1]),
position=ending_idx,
)
return True
def check_initial_combiner(label: str) -> bool:
"""Reject labels that begin with a combining mark.
Per :rfc:`5891` §4.2.3.2 a label must not start with a character of
Unicode general category ``M`` (Mark).
:param label: The label to check.
:returns: ``True`` if the first character is not a combining mark.
:raises IDNAError: If the label begins with a combining character.
"""
if label and unicodedata.category(label[0])[0] == "M":
raise IDNAError(
"Label begins with an illegal combining character",
code="leading_combiner",
text=label,
codepoint=ord(label[0]),
position=1,
)
return True
def check_hyphen_ok(label: str) -> bool:
"""Validate the hyphen restrictions for a label.
Per :rfc:`5891` §4.2.3.1 a label must not start or end with a hyphen
(``U+002D``), and must not have hyphens in both the third and fourth
positions (the prefix reserved for A-labels).
:param label: The label to check.
:returns: ``True`` if the hyphen restrictions are satisfied.
:raises IDNAError: If any of the hyphen restrictions are violated.
"""
if label[2:4] == "--":
raise IDNAError("Label has disallowed hyphens in 3rd and 4th position", code="hyphen_3_4")
if label.startswith("-") or label.endswith("-"):
raise IDNAError("Label must not start or end with a hyphen", code="hyphen_start_end")
return True
def check_nfc(label: str) -> None:
"""Require that a label is in Unicode Normalization Form C.
:param label: The label to check.
:raises IDNAError: If ``label`` differs from its NFC normalisation.
"""
if len(label) > _max_input_length:
raise IDNAError("Label too long", code="input_too_long")
if unicodedata.normalize("NFC", label) != label:
raise IDNAError("Label must be in Normalization Form C", code="not_nfc")
def valid_contextj(label: str, pos: int) -> bool:
"""Validate the CONTEXTJ rules from :rfc:`5892` Appendix A.
These rules govern the contextual use of the joiner codepoints
``U+200C`` (ZERO WIDTH NON-JOINER, Appendix A.1) and ``U+200D``
(ZERO WIDTH JOINER, Appendix A.2) within a label.
:param label: The label containing the codepoint.
:param pos: Index of the joiner codepoint within ``label``.
:returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTJ
rule, ``False`` otherwise (including when the codepoint at
``pos`` is not a recognised joiner).
:raises ValueError: If an adjacent codepoint has no Unicode name when
determining its combining class.
:raises IDNAError: If ``label`` exceeds the defensive input length limit.
"""
if len(label) > _max_input_length:
raise IDNAError("Label too long", code="input_too_long")
cp_value = ord(label[pos])
if cp_value == 0x200C:
if pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class:
return True
ok = False
for i in range(pos - 1, -1, -1):
joining_type = _joining_type(ord(label[i]))
if joining_type == "T":
continue
if joining_type in _bidi_joiner_l_or_d:
ok = True
break
break
if not ok:
return False
ok = False
for i in range(pos + 1, len(label)):
joining_type = _joining_type(ord(label[i]))
if joining_type == "T":
continue
if joining_type in _bidi_joiner_r_or_d:
ok = True
break
break
return ok
if cp_value == 0x200D:
return pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class
return False
def valid_contexto(label: str, pos: int, exception: bool = False) -> bool:
"""Validate the CONTEXTO rules from :rfc:`5892` Appendix A.
Covers the contextual rules for codepoints such as MIDDLE DOT
(``U+00B7``), Greek lower numeral sign, Hebrew punctuation, Katakana
middle dot, and the Arabic-Indic / Extended Arabic-Indic digit ranges.
:param label: The label containing the codepoint.
:param pos: Index of the codepoint within ``label``.
:param exception: Reserved for forward compatibility; currently unused.
:returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTO
rule, ``False`` otherwise (including when the codepoint is not a
recognised CONTEXTO codepoint).
:raises IDNAError: If ``label`` exceeds the defensive input length limit.
"""
if len(label) > _max_input_length:
raise IDNAError("Label too long", code="input_too_long")
cp_value = ord(label[pos])
if cp_value == 0x00B7:
return 0 < pos < len(label) - 1 and ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C
if cp_value == 0x0375:
if pos < len(label) - 1 and len(label) > 1:
return _is_script(label[pos + 1], "Greek")
return False
if cp_value in {0x05F3, 0x05F4}:
if pos > 0:
return _is_script(label[pos - 1], "Hebrew")
return False
if cp_value == 0x30FB:
for cp in label:
if cp == "\u30fb":
continue
if _is_script(cp, "Hiragana") or _is_script(cp, "Katakana") or _is_script(cp, "Han"):
return True
return False
if 0x660 <= cp_value <= 0x669:
return not any(0x6F0 <= ord(cp) <= 0x06F9 for cp in label)
if 0x6F0 <= cp_value <= 0x6F9:
return not any(0x660 <= ord(cp) <= 0x0669 for cp in label)
return False
def check_label(label: str | bytes | bytearray) -> None:
"""Run the full set of IDNA 2008 validity checks on a single label.
Applies, in order: NFC normalisation (:func:`check_nfc`), hyphen
restrictions (:func:`check_hyphen_ok`), the no-leading-combiner rule
(:func:`check_initial_combiner`), per-codepoint validity (PVALID,
CONTEXTJ, CONTEXTO classes from :rfc:`5892`), and the Bidi Rule
(:func:`check_bidi`).
:param label: The label to validate. ``bytes`` or ``bytearray`` input
is decoded as UTF-8 first.
:raises IDNAError: If the label is empty or fails a structural rule.
:raises InvalidCodepoint: If the label contains a DISALLOWED or
UNASSIGNED codepoint.
:raises InvalidCodepointContext: If a CONTEXTJ or CONTEXTO codepoint
is not valid in its context.
:raises IDNABidiError: If the Bidi Rule is violated.
"""
if len(label) > _max_input_length:
raise IDNAError("Label too long", code="input_too_long")
if isinstance(label, (bytes, bytearray)):
try:
label = label.decode("utf-8")
except UnicodeDecodeError as err:
raise IDNAError("Invalid UTF-8 in label", code="invalid_utf8") from err
if len(label) == 0:
raise IDNAError("Empty Label", code="empty_label")
# Check against the domain length rather than the label length to
# support some UTS #46 use cases, while still bounding the work done
# by the label contextual rules below.
if not valid_string_length(label, trailing_dot=True):
raise IDNAError("Label too long", code="label_too_long")
check_nfc(label)
check_hyphen_ok(label)
check_initial_combiner(label)
for pos, cp in enumerate(label):
cp_value = ord(cp)
if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]):
continue
if intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]):
try:
contextj_ok = valid_contextj(label, pos)
except ValueError as err:
raise IDNAError(
f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {label!r}",
code="unknown_codepoint",
text=label,
codepoint=cp_value,
position=pos + 1,
) from err
if not contextj_ok:
raise InvalidCodepointContext(
f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}",
code="contextj",
text=label,
codepoint=cp_value,
position=pos + 1,
)
elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]):
if not valid_contexto(label, pos):
raise InvalidCodepointContext(
f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}",
code="contexto",
text=label,
codepoint=cp_value,
position=pos + 1,
)
else:
raise InvalidCodepoint(
f"Codepoint {_unot(cp_value)} at position {pos + 1} of {label!r} not allowed",
code="disallowed_codepoint",
text=label,
codepoint=cp_value,
position=pos + 1,
)
check_bidi(label)
def alabel(label: str) -> bytes:
"""Convert a single U-label into its A-label form.
The result is the ASCII-Compatible Encoding (ACE) form per :rfc:`5891`
§4: the label is validated, Punycode-encoded, and prefixed with
``xn--``. Pure ASCII labels that are already valid IDNA labels are
returned unchanged (as :class:`bytes`).
:param label: The label to convert, as a Unicode string.
:returns: The A-label as ASCII-encoded :class:`bytes`.
:raises IDNAError: If the label is invalid or the resulting A-label
exceeds 63 octets.
"""
if len(label) > _max_input_length:
raise IDNAError("Label too long", code="input_too_long")
try:
label_bytes = label.encode("ascii")
except UnicodeEncodeError:
pass
else:
ulabel(label_bytes)
if not valid_label_length(label_bytes):
raise IDNAError("Label too long", code="label_too_long")
return label_bytes
check_label(label)
label_bytes = _alabel_prefix + _punycode(label)
if not valid_label_length(label_bytes):
raise IDNAError("Label too long", code="label_too_long")
return label_bytes
def ulabel(label: str | bytes | bytearray) -> str:
"""Convert a single A-label into its U-label form.
Performs the inverse of :func:`alabel`: an ``xn--``-prefixed label is
Punycode-decoded and validated, and is rejected unless it is the
canonical A-label for the decoded U-label (:rfc:`5891` §5.3). Labels
that are already Unicode (or plain ASCII without the ACE prefix) are
validated and returned as a Unicode string.
:param label: The label to convert. ``bytes`` or ``bytearray`` input
is treated as ASCII.
:returns: The U-label as a Unicode string.
:raises IDNAError: If the label is malformed or fails validation.
"""
if len(label) > _max_input_length:
raise IDNAError("Label too long", code="input_too_long")
if not isinstance(label, (bytes, bytearray)):
try:
label_bytes = label.encode("ascii")
except UnicodeEncodeError:
check_label(label)
return label
else:
label_bytes = bytes(label)
if not label_bytes.isascii():
raise IDNAError("Invalid ASCII in A-label", code="invalid_ascii")
label_bytes = label_bytes.lower()
if label_bytes.startswith(_alabel_prefix):
label_bytes = label_bytes[len(_alabel_prefix) :]
if not label_bytes:
raise IDNAError("Malformed A-label, no Punycode eligible content found", code="invalid_alabel")
if label_bytes.endswith(b"-"):
raise IDNAError("A-label must not end with a hyphen", code="invalid_alabel")
else:
check_label(label_bytes)
return label_bytes.decode("ascii")
try:
label = label_bytes.decode("punycode")
except UnicodeError as err:
raise IDNAError("Invalid A-label", code="invalid_alabel") from err
# RFC 5891 §5.3: the label is rejected unless re-encoding the decoded
# form reproduces the (lowercased) input. This catches "fake A-labels"
# (RFC 5890 §2.3.2.1) such as ``xn---bbk``, a non-canonical Punycode
# spelling of ``xn--bbk`` that would otherwise decode to the same
# U-label and so display identically to a different wire-format name.
if _punycode(label) != label_bytes:
raise IDNAError("A-label is not the canonical Punycode encoding of its U-label", code="non_canonical_alabel")
check_label(label)
return label
def _check_std3(text: str, domain: str, offset: int) -> None:
"""Raise if ``text``, a slice of ``domain`` starting at ``offset`` that
UTS #46 mapping left unchanged, contains an ASCII character disallowed
under ``UseSTD3ASCIIRules``."""
match = _std3_disallowed_re.search(text)
if match:
codepoint = ord(match.group())
position = offset + match.start() + 1
raise InvalidCodepoint(
f"Codepoint {_unot(codepoint)} not allowed at position {position} in {domain!r}",
code="uts46_std3",
text=domain,
codepoint=codepoint,
position=position,
)
def _warn_transitional() -> None:
warnings.warn(
"Transitional processing is deprecated in UTS #46 and has no effect. "
"The transitional argument will be removed in a future version.",
DeprecationWarning,
stacklevel=3,
)
def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str:
"""Apply the UTS #46 character mapping to a domain string.
Implements the mapping table from `UTS #46 §4
<https://www.unicode.org/reports/tr46/>`_: each character is kept,
replaced, or rejected based on its status (``V``, ``M``, ``D``,
``I``, ``X``). The result is returned in Normalisation Form C.
:param domain: The full domain name to remap.
:param std3_rules: If ``True``, apply UTS #46's ``UseSTD3ASCIIRules``:
after mapping, any ASCII character other than a lowercase letter,
digit, hyphen or the label separator ``.`` is rejected, whether it
appeared in the input or was produced by a mapping (e.g. U+FF01
FULLWIDTH EXCLAMATION MARK maps to ``!``). If ``False``, such
characters are passed through.
:param transitional: Deprecated and ignored. UTS #46 deprecated
transitional processing in Unicode 15.1 and deviation (status
``D``) codepoints are now always kept, so this has no effect
beyond emitting a :class:`DeprecationWarning`. It will be removed
in a future version.
:returns: The remapped domain, in Normalisation Form C.
:raises InvalidCodepoint: If the domain contains a disallowed
codepoint under the chosen rules.
:raises IDNAError: If ``domain`` exceeds the defensive input length limit.
"""
if transitional:
_warn_transitional()
if len(domain) > _max_input_length:
raise IDNAError("Domain too long", code="input_too_long")
if domain.isascii():
# The only ASCII mapping in UTS #46 is upper- to lowercase, and
# ASCII is invariant under NFC, so lowercasing is the whole job.
result = domain.lower()
if std3_rules:
_check_std3(result, domain, 0)
return result
from .uts46data import uts46_replacements, uts46_starts, uts46_statuses
# ``start`` marks the run of unchanged input not yet copied; a run is
# only sliced out when a character must be replaced or dropped, so the
# common no-change case makes no copy. STD3 is checked per output piece
# to report a violation at its input position.
output: list[str] = []
start = 0
for pos, char in enumerate(domain):
code_point = ord(char)
i = code_point if code_point < 256 else bisect.bisect_right(uts46_starts, code_point) - 1
status = uts46_statuses[i]
# UTS #46 §4: V valid, D deviation (kept), M mapped, I ignored,
# anything else disallowed.
if status == _STATUS_VALID:
continue
if status == _STATUS_MAPPED:
replacement = uts46_replacements[i]
elif status == _STATUS_DEVIATION:
continue
elif status == _STATUS_IGNORED:
replacement = None
else:
raise InvalidCodepoint(
f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}",
code="uts46_disallowed",
text=domain,
codepoint=code_point,
position=pos + 1,
)
if start < pos:
run = domain[start:pos]
if std3_rules:
_check_std3(run, domain, start)
output.append(run)
if replacement:
if std3_rules and _std3_disallowed_re.search(replacement):
raise InvalidCodepoint(
f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}",
code="uts46_std3",
text=domain,
codepoint=code_point,
position=pos + 1,
)
output.append(replacement)
start = pos + 1
if start == 0:
if std3_rules:
_check_std3(domain, domain, 0)
return unicodedata.normalize("NFC", domain)
tail = domain[start:]
if std3_rules:
_check_std3(tail, domain, start)
output.append(tail)
return unicodedata.normalize("NFC", "".join(output))
def encode(
s: str | bytes | bytearray,
strict: bool = False,
uts46: bool = False,
std3_rules: bool = False,
transitional: bool = False,
) -> bytes:
"""Encode a Unicode domain name into its ASCII (A-label) form.
Splits the input on label separators (only ``U+002E`` if ``strict`` is
set; otherwise also IDEOGRAPHIC FULL STOP ``U+3002``, FULLWIDTH FULL
STOP ``U+FF0E``, and HALFWIDTH IDEOGRAPHIC FULL STOP ``U+FF61``),
encodes each label with :func:`alabel`, and rejoins them with ``.``.
Optionally pre-processes the input through :func:`uts46_remap`.
:param s: The domain name to encode.
:param strict: If ``True``, only ``U+002E`` is recognised as a label
separator.
:param uts46: If ``True``, apply UTS #46 mapping before encoding.
:param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is
``True``.
:param transitional: Deprecated and ignored (see :func:`uts46_remap`):
emits a :class:`DeprecationWarning` and will be removed in a
future version.
:returns: The encoded domain as ASCII :class:`bytes`.
:raises IDNAError: If the domain is empty, contains an invalid label,
or exceeds the maximum domain length.
"""
if transitional:
_warn_transitional()
if not isinstance(s, str):
try:
s = str(s, "ascii")
except (UnicodeDecodeError, TypeError) as err:
raise IDNAError(
"should pass a unicode string to the function rather than a byte string.", code="invalid_ascii"
) from err
if len(s) > _max_input_length:
raise IDNAError("Domain too long", code="input_too_long")
if uts46:
s = uts46_remap(s, std3_rules)
if not valid_string_length(s, trailing_dot=True):
raise IDNAError("Domain too long", code="domain_too_long")
trailing_dot = False
result = []
labels = s.split(".") if strict else _unicode_dots_re.split(s)
if not labels or labels == [""]:
raise IDNAError("Empty domain", code="empty_domain")
if labels[-1] == "":
del labels[-1]
trailing_dot = True
for label in labels:
s = alabel(label)
if s:
result.append(s)
else:
raise IDNAError("Empty label", code="empty_label")
if trailing_dot:
result.append(b"")
s = b".".join(result)
if not valid_string_length(s, trailing_dot):
raise IDNAError("Domain too long", code="domain_too_long")
return s
def decode(
s: str | bytes | bytearray,
strict: bool = False,
uts46: bool = False,
std3_rules: bool = False,
display: bool = False,
) -> str:
"""Decode an A-label-encoded domain name back to Unicode.
Splits the input on label separators (see :func:`encode` for the
rules), decodes each label with :func:`ulabel`, and rejoins them
with ``.``. Optionally pre-processes the input through
:func:`uts46_remap`.
:param s: The domain name to decode.
:param strict: If ``True``, only ``U+002E`` is recognised as a label
separator.
:param uts46: If ``True``, apply UTS #46 mapping before decoding.
:param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is
``True``.
:param display: If ``True``, any ``xn--`` label that fails IDNA
validation is passed through unchanged (lowercased) rather than
aborting the whole call. Intended for "decode for display"
consumers (e.g. URL libraries, HTTP clients) that want to show
the user the label as it appears on the wire when it cannot be
rendered as Unicode. Matches the per-label recovery prescribed
by UTS #46 §4 and the WHATWG URL "domain to Unicode" algorithm.
:returns: The decoded domain as a Unicode string.
:raises IDNAError: If the input is not valid ASCII, contains an
invalid label, or is empty.
"""
if not isinstance(s, str):
try:
s = str(s, "ascii")
except (UnicodeDecodeError, TypeError) as err:
raise IDNAError("Invalid ASCII in A-label", code="invalid_ascii") from err
if len(s) > _max_input_length:
raise IDNAError("Domain too long", code="input_too_long")
if uts46:
s = uts46_remap(s, std3_rules, False)
if not valid_string_length(s, trailing_dot=True):
raise IDNAError("Domain too long", code="domain_too_long")
trailing_dot = False
result = []
labels = s.split(".") if strict else _unicode_dots_re.split(s)
if not labels or labels == [""]:
raise IDNAError("Empty domain", code="empty_domain")
if not labels[-1]:
del labels[-1]
trailing_dot = True
for label in labels:
try:
u = ulabel(label)
except IDNAError:
if display and label[:4].lower() == "xn--":
u = label.lower()
else:
raise
if u:
result.append(u)
else:
raise IDNAError("Empty label", code="empty_label")
if trailing_dot:
result.append("")
return ".".join(result)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,55 @@
"""
Given a list of integers, made up of (hopefully) a small number of long runs
of consecutive integers, compute a representation of the form
((start1, end1), (start2, end2) ...). Then answer the question "was x present
in the original list?" in time O(log(# runs)).
"""
import bisect
def intranges_from_list(list_: list[int]) -> tuple[int, ...]:
"""Represent a list of integers as a sequence of ranges:
((start_0, end_0), (start_1, end_1), ...), such that the original
integers are exactly those x such that start_i <= x < end_i for some i.
Ranges are encoded as single integers (start << 32 | end), not as tuples.
"""
sorted_list = sorted(list_)
ranges = []
last_write = -1
for i in range(len(sorted_list)):
if i + 1 < len(sorted_list) and sorted_list[i] == sorted_list[i + 1] - 1:
continue
current_range = sorted_list[last_write + 1 : i + 1]
ranges.append(_encode_range(current_range[0], current_range[-1] + 1))
last_write = i
return tuple(ranges)
def _encode_range(start: int, end: int) -> int:
return (start << 32) | end
def _decode_range(r: int) -> tuple[int, int]:
return (r >> 32), (r & ((1 << 32) - 1))
def intranges_contain(int_: int, ranges: tuple[int, ...]) -> bool:
"""Determine if `int_` falls into one of the ranges in `ranges`."""
tuple_ = _encode_range(int_, 0)
pos = bisect.bisect_left(ranges, tuple_)
# we could be immediately ahead of a tuple (start, end)
# with start < int_ <= end
if pos > 0:
left, right = _decode_range(ranges[pos - 1])
if left <= int_ < right:
return True
# or we could be immediately behind a tuple (int_, end)
if pos < len(ranges):
left, _ = _decode_range(ranges[pos])
if left == int_:
return True
return False
@@ -0,0 +1 @@
__version__ = "3.19"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,219 @@
# __
# /__) _ _ _ _ _/ _
# / ( (- (/ (/ (- _) / _)
# /
"""
Requests HTTP Library
~~~~~~~~~~~~~~~~~~~~~
Requests is an HTTP library, written in Python, for human beings.
Basic GET usage:
>>> import requests
>>> r = requests.get('https://www.python.org')
>>> r.status_code
200
>>> b'Python is a programming language' in r.content
True
... or POST:
>>> payload = dict(key1='value1', key2='value2')
>>> r = requests.post('https://httpbin.org/post', data=payload)
>>> print(r.text)
{
...
"form": {
"key1": "value1",
"key2": "value2"
},
...
}
The other HTTP methods are supported - see `requests.api`. Full documentation
is at <https://requests.readthedocs.io>.
:copyright: (c) 2017 by Kenneth Reitz.
:license: Apache 2.0, see LICENSE for more details.
"""
from __future__ import annotations
import warnings
import urllib3
from .exceptions import RequestsDependencyWarning
try:
from charset_normalizer import __version__ as charset_normalizer_version
except ImportError:
charset_normalizer_version = None
try:
from chardet import __version__ as chardet_version # type: ignore[import-not-found]
except ImportError:
chardet_version = None
def check_compatibility(
urllib3_version: str,
chardet_version: str | None,
charset_normalizer_version: str | None,
) -> None:
urllib3_version_list = urllib3_version.split(".")[:3]
assert urllib3_version_list != ["dev"] # Verify urllib3 isn't installed from git.
# Sometimes, urllib3 only reports its version as 16.1.
if len(urllib3_version_list) == 2:
urllib3_version_list.append("0")
# Check urllib3 for compatibility.
major, minor, patch = urllib3_version_list # noqa: F811
major, minor, patch = int(major), int(minor), int(patch)
# urllib3 >= 1.21.1
assert major >= 1
if major == 1:
assert minor >= 21
# Check charset_normalizer for compatibility.
if chardet_version:
major, minor, patch = chardet_version.split(".")[:3]
major, minor, patch = int(major), int(minor), int(patch)
# chardet_version >= 3.0.2, < 8.0.0
assert (3, 0, 2) <= (major, minor, patch) < (8, 0, 0)
elif charset_normalizer_version:
major, minor, patch = charset_normalizer_version.split(".")[:3]
major, minor, patch = int(major), int(minor), int(patch)
# charset_normalizer >= 2.0.0 < 4.0.0
assert (2, 0, 0) <= (major, minor, patch) < (4, 0, 0)
else:
warnings.warn(
"Unable to find acceptable character detection dependency "
"(chardet or charset_normalizer).",
RequestsDependencyWarning,
)
def _check_cryptography(cryptography_version: str) -> None:
# cryptography < 1.3.4
try:
cryptography_version_list = list(map(int, cryptography_version.split(".")))
except ValueError:
return
if cryptography_version_list < [1, 3, 4]:
warning = f"Old version of cryptography ({cryptography_version_list}) may cause slowdown."
warnings.warn(warning, RequestsDependencyWarning)
# Check imported dependencies for compatibility.
try:
check_compatibility(
urllib3.__version__, # type: ignore[reportPrivateImportUsage]
chardet_version, # type: ignore[reportUnknownArgumentType]
charset_normalizer_version,
)
except (AssertionError, ValueError):
warnings.warn(
f"urllib3 ({urllib3.__version__}) or chardet " # type: ignore[reportPrivateImportUsage]
f"({chardet_version})/charset_normalizer ({charset_normalizer_version}) "
"doesn't match a supported version!",
RequestsDependencyWarning,
)
# Attempt to enable urllib3's fallback for SNI support
# if the standard library doesn't support SNI or the
# 'ssl' library isn't available.
try:
try:
import ssl
except ImportError:
ssl = None
if not getattr(ssl, "HAS_SNI", False):
from urllib3.contrib import pyopenssl
pyopenssl.inject_into_urllib3()
# Check cryptography version
from cryptography import ( # type: ignore[reportMissingImports]
__version__ as cryptography_version, # type: ignore[reportUnknownVariableType]
)
_check_cryptography(cryptography_version) # type: ignore[reportUnknownArgumentType]
except ImportError:
pass
# urllib3's DependencyWarnings should be silenced.
from urllib3.exceptions import DependencyWarning
warnings.simplefilter("ignore", DependencyWarning)
# Set default logging handler to avoid "No handler found" warnings.
import logging
from logging import NullHandler
from . import packages, utils
from .__version__ import (
__author__,
__author_email__,
__build__,
__cake__,
__copyright__,
__description__,
__license__,
__title__,
__url__,
__version__,
)
from .api import delete, get, head, options, patch, post, put, request
from .exceptions import (
ConnectionError,
ConnectTimeout,
FileModeWarning,
HTTPError,
JSONDecodeError,
ReadTimeout,
RequestException,
Timeout,
TooManyRedirects,
URLRequired,
)
from .models import PreparedRequest, Request, Response
from .sessions import Session, session
from .status_codes import codes
__all__ = (
"ConnectionError",
"ConnectTimeout",
"HTTPError",
"JSONDecodeError",
"PreparedRequest",
"ReadTimeout",
"Request",
"RequestException",
"Response",
"Session",
"Timeout",
"TooManyRedirects",
"URLRequired",
"codes",
"delete",
"get",
"head",
"options",
"packages",
"patch",
"post",
"put",
"request",
"session",
"utils",
)
logging.getLogger(__name__).addHandler(NullHandler())
# FileModeWarnings go off per the default.
warnings.simplefilter("default", FileModeWarning, append=True)
@@ -0,0 +1,14 @@
# .-. .-. .-. . . .-. .-. .-. .-.
# |( |- |.| | | |- `-. | `-.
# ' ' `-' `-`.`-' `-' `-' ' `-'
__title__ = "requests"
__description__ = "Python HTTP for Humans."
__url__ = "https://requests.readthedocs.io"
__version__ = "2.34.2"
__build__ = 0x023402
__author__ = "Kenneth Reitz"
__author_email__ = "[email protected]"
__license__ = "Apache-2.0"
__copyright__ = "Copyright Kenneth Reitz"
__cake__ = "\u2728 \U0001f370 \u2728"
@@ -0,0 +1,51 @@
"""
requests._internal_utils
~~~~~~~~~~~~~~
Provides utility functions that are consumed internally by Requests
which depend on extremely few external helpers (such as compat)
"""
import re
from .compat import builtin_str
_VALID_HEADER_NAME_RE_BYTE = re.compile(rb"^[^:\s][^:\r\n]*\Z")
_VALID_HEADER_NAME_RE_STR = re.compile(r"^[^:\s][^:\r\n]*\Z")
_VALID_HEADER_VALUE_RE_BYTE = re.compile(rb"^\S[^\r\n]*\Z|^\Z")
_VALID_HEADER_VALUE_RE_STR = re.compile(r"^\S[^\r\n]*\Z|^\Z")
_HEADER_VALIDATORS_STR = (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR)
_HEADER_VALIDATORS_BYTE = (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE)
HEADER_VALIDATORS = {
bytes: _HEADER_VALIDATORS_BYTE,
str: _HEADER_VALIDATORS_STR,
}
def to_native_string(string: str | bytes, encoding: str = "ascii") -> str:
"""Given a string object, regardless of type, returns a representation of
that string in the native string type, encoding and decoding where
necessary. This assumes ASCII unless told otherwise.
"""
if isinstance(string, builtin_str):
out = string
else:
out = string.decode(encoding)
return out
def unicode_is_ascii(u_string: str) -> bool:
"""Determine if unicode string only contains ASCII characters.
:param str u_string: unicode string to check. Must be unicode
and not Python 2 `str`.
:rtype: bool
"""
assert isinstance(u_string, str)
try:
u_string.encode("ascii")
return True
except UnicodeEncodeError:
return False
@@ -0,0 +1,183 @@
"""
requests._types
~~~~~~~~~~~~~~~
This module contains type aliases used internally by the Requests library.
These types are not part of the public API and must not be relied upon
by external code.
"""
from __future__ import annotations
from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence
from typing import (
TYPE_CHECKING,
Any,
Protocol,
TypeAlias,
TypeVar,
runtime_checkable,
)
_T_co = TypeVar("_T_co", covariant=True)
_KT_co = TypeVar("_KT_co", covariant=True)
_VT_co = TypeVar("_VT_co", covariant=True)
@runtime_checkable
class SupportsRead(Protocol[_T_co]):
def read(self, length: int = ..., /) -> _T_co: ...
@runtime_checkable
class SupportsItems(Protocol[_KT_co, _VT_co]):
def items(self) -> Iterable[tuple[_KT_co, _VT_co]]: ...
# These are needed at runtime for default_hooks() return type
HookType: TypeAlias = Callable[["Response"], Any]
HooksInputType: TypeAlias = Mapping[str, Iterable[HookType] | HookType]
def is_prepared(request: PreparedRequest) -> TypeIs[_ValidatedRequest]:
"""Verify a PreparedRequest has been fully prepared."""
if TYPE_CHECKING:
return request.url is not None and request.method is not None
# noop at runtime to avoid AssertionError
return True
if TYPE_CHECKING:
from http.cookiejar import CookieJar
from typing import TypeAlias, TypedDict
from typing_extensions import (
Buffer, # TODO: move to collections.abc when Python >= 3.12
TypeIs, # TODO: move to typing when Python >= 3.13
)
from .auth import AuthBase
from .cookies import RequestsCookieJar
from .models import PreparedRequest, Response
from .structures import CaseInsensitiveDict
class _ValidatedRequest(PreparedRequest):
"""Subtype asserting a PreparedRequest has been fully prepared before calling.
The override suppression is required because mutable attribute types are
invariant (Liskov), but we only narrow after preparation is complete. This
is the explicit contract for Requests but Python's typing doesn't have a
better way to represent the requirement.
"""
url: str # type: ignore[reportIncompatibleVariableOverride]
method: str # type: ignore[reportIncompatibleVariableOverride]
# Type aliases for core API concepts (ordered by request() signature)
UriType: TypeAlias = str | bytes
_ParamsMappingKeyType: TypeAlias = str | bytes | int | float
_ParamsMappingValueType: TypeAlias = (
str | bytes | int | float | Iterable[str | bytes | int | float] | None
)
ParamsType: TypeAlias = (
SupportsItems[_ParamsMappingKeyType, _ParamsMappingValueType]
| tuple[tuple[_ParamsMappingKeyType, _ParamsMappingValueType], ...]
| Iterable[tuple[_ParamsMappingKeyType, _ParamsMappingValueType]]
| str
| bytes
| None
)
KVDataType: TypeAlias = Iterable[tuple[Any, Any]] | SupportsItems[Any, Any]
RawDataType: TypeAlias = KVDataType | str | bytes
StreamDataType: TypeAlias = SupportsRead[str | bytes]
EncodableDataType: TypeAlias = RawDataType | StreamDataType
DataType: TypeAlias = (
KVDataType
| Iterable[bytes | str]
| str
| bytes
| Buffer
| SupportsRead[str | bytes]
| None
)
BodyType: TypeAlias = (
bytes | str | Iterable[bytes | str] | SupportsRead[bytes | str] | None
)
HeadersType: TypeAlias = Mapping[str, str | bytes] | None
CookiesType: TypeAlias = RequestsCookieJar | Mapping[str, str]
# Building blocks for FilesType
_FileName: TypeAlias = str | None
_FileContent: TypeAlias = SupportsRead[str | bytes] | str | bytes
_FileSpecBasic: TypeAlias = tuple[_FileName, _FileContent]
_FileSpecWithContentType: TypeAlias = tuple[_FileName, _FileContent, str]
_FileSpecWithHeaders: TypeAlias = tuple[
_FileName, _FileContent, str, CaseInsensitiveDict[str] | Mapping[str, str]
]
_FileSpec: TypeAlias = (
_FileContent | _FileSpecBasic | _FileSpecWithContentType | _FileSpecWithHeaders
)
FilesType: TypeAlias = (
Mapping[str, _FileSpec] | Iterable[tuple[str, _FileSpec]] | None
)
AuthType: TypeAlias = (
tuple[str, str] | AuthBase | Callable[[PreparedRequest], PreparedRequest] | None
)
TimeoutType: TypeAlias = float | tuple[float | None, float | None] | None
ProxiesType: TypeAlias = MutableMapping[str, str]
HooksType: TypeAlias = dict[str, list[HookType]] | None
VerifyType: TypeAlias = bool | str
CertType: TypeAlias = str | tuple[str, str] | None
JsonType: TypeAlias = (
None
| bool
| int
| float
| str
| Sequence["JsonType"]
| Mapping[str, "JsonType"]
)
# TypedDicts for Unpack kwargs (PEP 692)
class BaseRequestKwargs(TypedDict, total=False):
headers: HeadersType
cookies: RequestsCookieJar | CookieJar | dict[str, str] | None
files: FilesType
auth: AuthType
timeout: TimeoutType
allow_redirects: bool
proxies: dict[str, str] | None
hooks: HooksInputType | None
stream: bool | None
verify: VerifyType | None
cert: CertType
class RequestKwargs(BaseRequestKwargs, total=False):
"""kwargs for request(), options(), head(), delete()."""
params: ParamsType
data: DataType
json: JsonType
class GetKwargs(BaseRequestKwargs, total=False):
data: DataType
json: JsonType
class PostKwargs(BaseRequestKwargs, total=False):
params: ParamsType
class DataKwargs(BaseRequestKwargs, total=False):
"""kwargs for put(), patch()."""
params: ParamsType
json: JsonType
@@ -0,0 +1,748 @@
"""
requests.adapters
~~~~~~~~~~~~~~~~~
This module contains the transport adapters that Requests uses to define
and maintain connections.
"""
from __future__ import annotations
import os.path
import socket # noqa: F401 # type: ignore[reportUnusedImport]
import typing
import warnings
from typing import Any
from urllib3.exceptions import (
ClosedPoolError,
ConnectTimeoutError,
LocationValueError,
MaxRetryError,
NewConnectionError,
ProtocolError,
ReadTimeoutError,
ResponseError,
)
from urllib3.exceptions import HTTPError as _HTTPError
from urllib3.exceptions import InvalidHeader as _InvalidHeader
from urllib3.exceptions import ProxyError as _ProxyError
from urllib3.exceptions import SSLError as _SSLError
from urllib3.poolmanager import PoolManager, proxy_from_url
from urllib3.util import Timeout as TimeoutSauce
from urllib3.util import parse_url
from urllib3.util.retry import Retry
from .auth import _basic_auth_str # type: ignore[reportPrivateUsage]
from .compat import basestring, urlparse
from .cookies import extract_cookies_to_jar
from .exceptions import (
ConnectionError,
ConnectTimeout,
InvalidHeader,
InvalidProxyURL,
InvalidSchema,
InvalidURL,
ProxyError,
ReadTimeout,
RetryError,
SSLError,
)
from .models import Response
from .structures import CaseInsensitiveDict
from .utils import (
DEFAULT_CA_BUNDLE_PATH,
get_auth_from_url,
get_encoding_from_headers,
prepend_scheme_if_needed,
select_proxy,
urldefragauth,
)
try:
from urllib3.contrib.socks import SOCKSProxyManager # type: ignore[assignment]
except ImportError:
def SOCKSProxyManager(*args: Any, **kwargs: Any) -> None:
raise InvalidSchema("Missing dependencies for SOCKS support.")
if typing.TYPE_CHECKING:
from urllib3.connectionpool import HTTPConnectionPool
from urllib3.poolmanager import PoolManager as _PoolManager
from . import _types as _t
from .models import PreparedRequest
from ._types import is_prepared as _is_prepared
DEFAULT_POOLBLOCK = False
DEFAULT_POOLSIZE = 10
DEFAULT_RETRIES = 0
DEFAULT_POOL_TIMEOUT = None
def _urllib3_request_context(
request: PreparedRequest,
verify: bool | str | None,
client_cert: tuple[str, str] | str | None,
poolmanager: PoolManager,
) -> tuple[dict[str, Any], dict[str, Any]]:
host_params: dict[str, Any] = {}
pool_kwargs: dict[str, Any] = {}
parsed_request_url = urlparse(request.url)
scheme = parsed_request_url.scheme.lower()
port = parsed_request_url.port
cert_reqs = "CERT_REQUIRED"
if verify is False:
cert_reqs = "CERT_NONE"
elif isinstance(verify, str):
if not os.path.isdir(verify):
pool_kwargs["ca_certs"] = verify
else:
pool_kwargs["ca_cert_dir"] = verify
pool_kwargs["cert_reqs"] = cert_reqs
if client_cert is not None:
if isinstance(client_cert, tuple) and len(client_cert) == 2:
pool_kwargs["cert_file"] = client_cert[0]
pool_kwargs["key_file"] = client_cert[1]
else:
# According to our docs, we allow users to specify just the client
# cert path
pool_kwargs["cert_file"] = client_cert
host_params = {
"scheme": scheme,
"host": parsed_request_url.hostname,
"port": port,
}
return host_params, pool_kwargs
class BaseAdapter:
"""The Base Transport Adapter"""
def __init__(self) -> None:
super().__init__()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param verify: (optional) Either a boolean, in which case it controls whether we verify
the server's TLS certificate, or a string, in which case it must be a path
to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
"""
raise NotImplementedError
def close(self) -> None:
"""Cleans up adapter specific items."""
raise NotImplementedError
class HTTPAdapter(BaseAdapter):
"""The built-in HTTP Adapter for urllib3.
Provides a general-case interface for Requests sessions to contact HTTP and
HTTPS urls by implementing the Transport Adapter interface. This class will
usually be created by the :class:`Session <Session>` class under the
covers.
:param pool_connections: The number of urllib3 connection pools to cache.
:param pool_maxsize: The maximum number of connections to save in the pool.
:param max_retries: The maximum number of retries each connection
should attempt. Note, this applies only to failed DNS lookups, socket
connections and connection timeouts, never to requests where data has
made it to the server. By default, Requests does not retry failed
connections. If you need granular control over the conditions under
which we retry a request, import urllib3's ``Retry`` class and pass
that instead.
:param pool_block: Whether the connection pool should block for connections.
Usage::
>>> import requests
>>> s = requests.Session()
>>> a = requests.adapters.HTTPAdapter(max_retries=3)
>>> s.mount('http://', a)
"""
__attrs__: list[str] = [
"max_retries",
"config",
"_pool_connections",
"_pool_maxsize",
"_pool_block",
]
max_retries: Retry
config: dict[str, Any]
proxy_manager: dict[str, Any]
_pool_connections: int
_pool_maxsize: int
_pool_block: bool
poolmanager: _PoolManager
def __init__(
self,
pool_connections: int = DEFAULT_POOLSIZE,
pool_maxsize: int = DEFAULT_POOLSIZE,
max_retries: int | Retry = DEFAULT_RETRIES,
pool_block: bool = DEFAULT_POOLBLOCK,
) -> None:
if max_retries == DEFAULT_RETRIES:
self.max_retries = Retry(0, read=False)
else:
self.max_retries = Retry.from_int(max_retries)
self.config = {}
self.proxy_manager = {}
super().__init__()
self._pool_connections = pool_connections
self._pool_maxsize = pool_maxsize
self._pool_block = pool_block
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
def __getstate__(self) -> dict[str, Any]:
return {attr: getattr(self, attr, None) for attr in self.__attrs__}
def __setstate__(self, state: dict[str, Any]) -> None:
# Can't handle by adding 'proxy_manager' to self.__attrs__ because
# self.poolmanager uses a lambda function, which isn't pickleable.
self.proxy_manager = {}
self.config = {}
for attr, value in state.items():
setattr(self, attr, value)
self.init_poolmanager(
self._pool_connections, self._pool_maxsize, block=self._pool_block
)
def init_poolmanager(
self,
connections: int,
maxsize: int,
block: bool = DEFAULT_POOLBLOCK,
**pool_kwargs: Any,
) -> None:
"""Initializes a urllib3 PoolManager.
This method should not be called from user code, and is only
exposed for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param connections: The number of urllib3 connection pools to cache.
:param maxsize: The maximum number of connections to save in the pool.
:param block: Block when no free connections are available.
:param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
"""
# save these values for pickling
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
**pool_kwargs,
)
def proxy_manager_for(self, proxy: str, **proxy_kwargs: Any) -> Any:
"""Return urllib3 ProxyManager for the given proxy.
This method should not be called from user code, and is only
exposed for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param proxy: The proxy to return a urllib3 ProxyManager for.
:param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
:returns: ProxyManager
:rtype: urllib3.ProxyManager
"""
if proxy in self.proxy_manager:
manager = self.proxy_manager[proxy]
elif proxy.lower().startswith("socks"):
username, password = get_auth_from_url(proxy)
manager = self.proxy_manager[proxy] = SOCKSProxyManager(
proxy,
username=username,
password=password,
num_pools=self._pool_connections,
maxsize=self._pool_maxsize,
block=self._pool_block,
**proxy_kwargs,
)
else:
proxy_headers = self.proxy_headers(proxy)
manager = self.proxy_manager[proxy] = proxy_from_url(
proxy,
proxy_headers=proxy_headers,
num_pools=self._pool_connections,
maxsize=self._pool_maxsize,
block=self._pool_block,
**proxy_kwargs,
)
return manager
def cert_verify(
self, conn: Any, url: str, verify: _t.VerifyType, cert: _t.CertType
) -> None:
"""Verify a SSL certificate. This method should not be called from user
code, and is only exposed for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param conn: The urllib3 connection object associated with the cert.
:param url: The requested URL.
:param verify: Either a boolean, in which case it controls whether we verify
the server's TLS certificate, or a string, in which case it must be a path
to a CA bundle to use
:param cert: The SSL certificate to verify.
"""
if url.lower().startswith("https") and verify:
cert_loc = None
# Allow self-specified cert location.
if verify is not True:
cert_loc = verify
if not cert_loc:
cert_loc = DEFAULT_CA_BUNDLE_PATH
if not cert_loc or not os.path.exists(cert_loc):
raise OSError(
f"Could not find a suitable TLS CA certificate bundle, "
f"invalid path: {cert_loc}"
)
conn.cert_reqs = "CERT_REQUIRED"
if not os.path.isdir(cert_loc):
conn.ca_certs = cert_loc
else:
conn.ca_cert_dir = cert_loc
else:
conn.cert_reqs = "CERT_NONE"
conn.ca_certs = None
conn.ca_cert_dir = None
if cert:
if not isinstance(cert, basestring):
conn.cert_file = cert[0]
conn.key_file = cert[1]
else:
conn.cert_file = cert
conn.key_file = None
if conn.cert_file and not os.path.exists(conn.cert_file):
raise OSError(
f"Could not find the TLS certificate file, "
f"invalid path: {conn.cert_file}"
)
if conn.key_file and not os.path.exists(conn.key_file):
raise OSError(
f"Could not find the TLS key file, invalid path: {conn.key_file}"
)
def build_response(self, req: PreparedRequest, resp: Any) -> Response:
"""Builds a :class:`Response <requests.Response>` object from a urllib3
response. This should not be called from user code, and is only exposed
for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
:param resp: The urllib3 response object.
:rtype: requests.Response
"""
assert _is_prepared(req)
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, "status", None) # type: ignore[assignment]
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode("utf-8")
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, resp)
# Give the Response some context.
response.request = req
response.connection = self
return response
def build_connection_pool_key_attributes(
self, request: PreparedRequest, verify: _t.VerifyType, cert: _t.CertType = None
) -> tuple[dict[str, Any], dict[str, Any]]:
"""Build the PoolKey attributes used by urllib3 to return a connection.
This looks at the PreparedRequest, the user-specified verify value,
and the value of the cert parameter to determine what PoolKey values
to use to select a connection from a given urllib3 Connection Pool.
The SSL related pool key arguments are not consistently set. As of
this writing, use the following to determine what keys may be in that
dictionary:
* If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the
default Requests SSL Context
* If ``verify`` is ``False``, ``"ssl_context"`` will not be set but
``"cert_reqs"`` will be set
* If ``verify`` is a string, (i.e., it is a user-specified trust bundle)
``"ca_certs"`` will be set if the string is not a directory recognized
by :py:func:`os.path.isdir`, otherwise ``"ca_cert_dir"`` will be
set.
* If ``"cert"`` is specified, ``"cert_file"`` will always be set. If
``"cert"`` is a tuple with a second item, ``"key_file"`` will also
be present
To override these settings, one may subclass this class, call this
method and use the above logic to change parameters as desired. For
example, if one wishes to use a custom :py:class:`ssl.SSLContext` one
must both set ``"ssl_context"`` and based on what else they require,
alter the other keys to ensure the desired behaviour.
:param request:
The PreparedRequest being sent over the connection.
:type request:
:class:`~requests.models.PreparedRequest`
:param verify:
Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use.
:param cert:
(optional) Any user-provided SSL certificate for client
authentication (a.k.a., mTLS). This may be a string (i.e., just
the path to a file which holds both certificate and key) or a
tuple of length 2 with the certificate file path and key file
path.
:returns:
A tuple of two dictionaries. The first is the "host parameters"
portion of the Pool Key including scheme, hostname, and port. The
second is a dictionary of SSLContext related parameters.
"""
return _urllib3_request_context(request, verify, cert, self.poolmanager)
def get_connection_with_tls_context(
self,
request: PreparedRequest,
verify: _t.VerifyType,
proxies: dict[str, str] | None = None,
cert: _t.CertType = None,
) -> HTTPConnectionPool:
"""Returns a urllib3 connection for the given request and TLS settings.
This should not be called from user code, and is only exposed for use
when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param request:
The :class:`PreparedRequest <PreparedRequest>` object to be sent
over the connection.
:param verify:
Either a boolean, in which case it controls whether we verify the
server's TLS certificate, or a string, in which case it must be a
path to a CA bundle to use.
:param proxies:
(optional) The proxies dictionary to apply to the request.
:param cert:
(optional) Any user-provided SSL certificate to be used for client
authentication (a.k.a., mTLS).
:rtype:
urllib3.HTTPConnectionPool
"""
assert _is_prepared(request)
proxy = select_proxy(request.url, proxies)
try:
host_params, pool_kwargs = self.build_connection_pool_key_attributes(
request,
verify,
cert,
)
except ValueError as e:
raise InvalidURL(e, request=request)
if proxy:
proxy = prepend_scheme_if_needed(proxy, "http")
proxy_url = parse_url(proxy)
if not proxy_url.host:
raise InvalidProxyURL(
"Please check proxy URL. It is malformed "
"and could be missing the host."
)
proxy_manager = self.proxy_manager_for(proxy)
conn = proxy_manager.connection_from_host(
**host_params, pool_kwargs=pool_kwargs
)
else:
# Only scheme should be lower case
conn = self.poolmanager.connection_from_host(
**host_params, pool_kwargs=pool_kwargs
)
return conn
def get_connection(
self, url: str, proxies: dict[str, str] | None = None
) -> HTTPConnectionPool:
"""DEPRECATED: Users should move to `get_connection_with_tls_context`
for all subclasses of HTTPAdapter using Requests>=2.32.2.
Returns a urllib3 connection for the given URL. This should not be
called from user code, and is only exposed for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param url: The URL to connect to.
:param proxies: (optional) A Requests-style dictionary of proxies used on this request.
:rtype: urllib3.HTTPConnectionPool
"""
warnings.warn(
(
"`get_connection` has been deprecated in favor of "
"`get_connection_with_tls_context`. Custom HTTPAdapter subclasses "
"will need to migrate for Requests>=2.32.2. Please see "
"https://github.com/psf/requests/pull/6710 for more details."
),
DeprecationWarning,
)
proxy = select_proxy(url, proxies)
if proxy:
proxy = prepend_scheme_if_needed(proxy, "http")
proxy_url = parse_url(proxy)
if not proxy_url.host:
raise InvalidProxyURL(
"Please check proxy URL. It is malformed "
"and could be missing the host."
)
proxy_manager = self.proxy_manager_for(proxy)
conn = proxy_manager.connection_from_url(url)
else:
# Only scheme should be lower case
parsed = urlparse(url)
url = parsed.geturl()
conn = self.poolmanager.connection_from_url(url)
return conn
def close(self) -> None:
"""Disposes of any internal state.
Currently, this closes the PoolManager and any active ProxyManager,
which closes any pooled connections.
"""
self.poolmanager.clear()
for proxy in self.proxy_manager.values():
proxy.clear()
def request_url(
self, request: PreparedRequest, proxies: dict[str, str] | None
) -> str:
"""Obtain the url to use when making the final request.
If the message is being sent through a HTTP proxy, the full URL has to
be used. Otherwise, we should only use the path portion of the URL.
This should not be called from user code, and is only exposed for use
when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
:rtype: str
"""
assert _is_prepared(request)
proxy = select_proxy(request.url, proxies)
scheme = urlparse(request.url).scheme
is_proxied_http_request = proxy and scheme != "https"
using_socks_proxy = False
if proxy:
proxy_scheme = urlparse(proxy).scheme.lower()
using_socks_proxy = proxy_scheme.startswith("socks")
url = request.path_url
if is_proxied_http_request and not using_socks_proxy:
url = urldefragauth(request.url)
return url
def add_headers(self, request: PreparedRequest, **kwargs: Any) -> None:
"""Add any headers needed by the connection. As of v2.0 this does
nothing by default, but is left for overriding by users that subclass
the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
This should not be called from user code, and is only exposed for use
when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
:param kwargs: The keyword arguments from the call to send().
"""
pass
def proxy_headers(self, proxy: str) -> dict[str, str]:
"""Returns a dictionary of the headers to add to any request sent
through a proxy. This works with urllib3 magic to ensure that they are
correctly sent to the proxy, rather than in a tunnelled request if
CONNECT is being used.
This should not be called from user code, and is only exposed for use
when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param proxy: The url of the proxy being used for this request.
:rtype: dict
"""
headers: dict[str, str] = {}
username, password = get_auth_from_url(proxy)
if username:
headers["Proxy-Authorization"] = _basic_auth_str(username, password)
return headers
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
raise SSLError(e, request=request)
raise ConnectionError(e, request=request)
except ClosedPoolError as e:
raise ConnectionError(e, request=request)
except _ProxyError as e:
raise ProxyError(e)
except (_SSLError, _HTTPError) as e:
if isinstance(e, _SSLError):
# This branch is for urllib3 versions earlier than v1.22
raise SSLError(e, request=request)
elif isinstance(e, ReadTimeoutError):
raise ReadTimeout(e, request=request)
elif isinstance(e, _InvalidHeader):
raise InvalidHeader(e, request=request)
else:
raise
return self.build_response(request, resp)
@@ -0,0 +1,180 @@
"""
requests.api
~~~~~~~~~~~~
This module implements the Requests API.
:copyright: (c) 2012 by Kenneth Reitz.
:license: Apache2, see LICENSE for more details.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from . import sessions
from .models import Response
if TYPE_CHECKING:
from typing_extensions import Unpack
from . import _types as _t
def request(
method: str, url: _t.UriType, **kwargs: Unpack[_t.RequestKwargs]
) -> Response:
"""Constructs and sends a :class:`Request <Request>`.
:param method: method for the new :class:`Request` object: ``GET``, ``OPTIONS``, ``HEAD``, ``POST``, ``PUT``, ``PATCH``, or ``DELETE``.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary, list of tuples or bytes to send
in the query string for the :class:`Request`.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload.
``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')``
or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content_type'`` is a string
defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers
to add for the file.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) How many seconds to wait for the server to send data
before giving up, as a float, or a :ref:`(connect timeout, read
timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param verify: (optional) Either a boolean, in which case it controls whether we verify
the server's TLS certificate, or a string, in which case it must be a path
to a CA bundle to use. Defaults to ``True``.
:param stream: (optional) if ``False``, the response content will be immediately downloaded.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
:return: :class:`Response <Response>` object
:rtype: requests.Response
Usage::
>>> import requests
>>> req = requests.request('GET', 'https://httpbin.org/get')
>>> req
<Response [200]>
"""
# By using the 'with' statement we are sure the session is closed, thus we
# avoid leaving sockets open which can trigger a ResourceWarning in some
# cases, and look like a memory leak in others.
with sessions.Session() as session:
return session.request(method=method, url=url, **kwargs)
def get(
url: _t.UriType, params: _t.ParamsType = None, **kwargs: Unpack[_t.GetKwargs]
) -> Response:
r"""Sends a GET request.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary, list of tuples or bytes to send
in the query string for the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("get", url, params=params, **kwargs)
def options(url: _t.UriType, **kwargs: Unpack[_t.RequestKwargs]) -> Response:
r"""Sends an OPTIONS request.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("options", url, **kwargs)
def head(url: _t.UriType, **kwargs: Unpack[_t.RequestKwargs]) -> Response:
r"""Sends a HEAD request.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes. If
`allow_redirects` is not provided, it will be set to `False` (as
opposed to the default :meth:`request` behavior).
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", False)
return request("head", url, **kwargs)
def post(
url: _t.UriType,
data: _t.DataType = None,
json: _t.JsonType = None,
**kwargs: Unpack[_t.PostKwargs],
) -> Response:
r"""Sends a POST request.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("post", url, data=data, json=json, **kwargs)
def put(
url: _t.UriType, data: _t.DataType = None, **kwargs: Unpack[_t.DataKwargs]
) -> Response:
r"""Sends a PUT request.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("put", url, data=data, **kwargs)
def patch(
url: _t.UriType, data: _t.DataType = None, **kwargs: Unpack[_t.DataKwargs]
) -> Response:
r"""Sends a PATCH request.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("patch", url, data=data, **kwargs)
def delete(url: _t.UriType, **kwargs: Unpack[_t.RequestKwargs]) -> Response:
r"""Sends a DELETE request.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("delete", url, **kwargs)
@@ -0,0 +1,354 @@
"""
requests.auth
~~~~~~~~~~~~~
This module contains the authentication handlers for Requests.
"""
from __future__ import annotations
import hashlib
import os
import re
import threading
import time
import warnings
from base64 import b64encode
from typing import TYPE_CHECKING, Any, Final, cast, overload
from ._internal_utils import to_native_string
from .compat import basestring, str, urlparse
from .cookies import extract_cookies_to_jar
from .utils import parse_dict_header
if TYPE_CHECKING:
from http.cookiejar import CookieJar
from typing import Any
from .models import PreparedRequest, Response
CONTENT_TYPE_FORM_URLENCODED: Final = "application/x-www-form-urlencoded"
CONTENT_TYPE_MULTI_PART: Final = "multipart/form-data"
def _basic_auth_str(username: bytes | str, password: bytes | str) -> str:
"""Returns a Basic Auth string."""
# "I want us to put a big-ol' comment on top of it that
# says that this behaviour is dumb but we need to preserve
# it because people are relying on it."
# - Lukasa
#
# These are here solely to maintain backwards compatibility
# for things like ints. This will be removed in 3.0.0.
if not isinstance(username, basestring): # type: ignore[reportUnnecessaryIsInstance] # runtime guard for non-str/bytes
warnings.warn(
"Non-string usernames will no longer be supported in Requests "
f"3.0.0. Please convert the object you've passed in ({username!r}) to "
"a string or bytes object in the near future to avoid "
"problems.",
category=DeprecationWarning,
)
username = str(username)
if not isinstance(password, basestring): # type: ignore[reportUnnecessaryIsInstance] # runtime guard for non-str/bytes
warnings.warn(
"Non-string passwords will no longer be supported in Requests "
f"3.0.0. Please convert the object you've passed in ({type(password)!r}) to "
"a string or bytes object in the near future to avoid "
"problems.",
category=DeprecationWarning,
)
password = str(password)
# -- End Removal --
if isinstance(username, str):
username = username.encode("latin1")
if isinstance(password, str):
password = password.encode("latin1")
authstr = "Basic " + to_native_string(
b64encode(b":".join((username, password))).strip()
)
return authstr
class AuthBase:
"""Base class that all auth implementations derive from"""
def __call__(self, r: PreparedRequest) -> PreparedRequest:
raise NotImplementedError("Auth hooks must be callable.")
class HTTPBasicAuth(AuthBase):
"""Attaches HTTP Basic Authentication to the given Request object."""
username: bytes | str
password: bytes | str
@overload
def __init__(self, username: str, password: str) -> None: ...
@overload
def __init__(self, username: bytes, password: bytes) -> None: ...
def __init__(self, username: bytes | str, password: bytes | str) -> None:
self.username = username
self.password = password
def __eq__(self, other: object) -> bool:
return all(
[
self.username == getattr(other, "username", None),
self.password == getattr(other, "password", None),
]
)
def __ne__(self, other: Any) -> bool:
return not self == other
def __call__(self, r: PreparedRequest) -> PreparedRequest:
r.headers["Authorization"] = _basic_auth_str(self.username, self.password)
return r
class HTTPProxyAuth(HTTPBasicAuth):
"""Attaches HTTP Proxy Authentication to a given Request object."""
def __call__(self, r: PreparedRequest) -> PreparedRequest:
r.headers["Proxy-Authorization"] = _basic_auth_str(self.username, self.password)
return r
class HTTPDigestAuth(AuthBase):
"""Attaches HTTP Digest Authentication to the given Request object."""
username: bytes | str
password: bytes | str
_thread_local: threading.local
last_nonce: str
nonce_count: int
chal: dict[str, str]
pos: int | None
num_401_calls: int | None
@overload
def __init__(self, username: str, password: str) -> None: ...
@overload
def __init__(self, username: bytes, password: bytes) -> None: ...
def __init__(self, username: bytes | str, password: bytes | str) -> None:
self.username = username
self.password = password
# Keep state in per-thread local storage
self._thread_local = threading.local()
def init_per_thread_state(self) -> None:
# Ensure state is initialized just once per-thread
if not hasattr(self._thread_local, "init"):
self._thread_local.init = True
self._thread_local.last_nonce = ""
self._thread_local.nonce_count = 0
self._thread_local.chal = {}
self._thread_local.pos = None
self._thread_local.num_401_calls = None
def build_digest_header(self, method: str, url: str) -> str | None:
"""
:rtype: str
"""
realm = self._thread_local.chal["realm"]
nonce = self._thread_local.chal["nonce"]
qop = self._thread_local.chal.get("qop")
algorithm = self._thread_local.chal.get("algorithm")
opaque = self._thread_local.chal.get("opaque")
hash_utf8 = None
if algorithm is None:
_algorithm = "MD5"
else:
_algorithm = algorithm.upper()
# lambdas assume digest modules are imported at the top level
if _algorithm == "MD5" or _algorithm == "MD5-SESS":
def md5_utf8(x: str | bytes) -> str:
if isinstance(x, str):
x = x.encode("utf-8")
return hashlib.md5(x, usedforsecurity=False).hexdigest()
hash_utf8 = md5_utf8
elif _algorithm == "SHA":
def sha_utf8(x: str | bytes) -> str:
if isinstance(x, str):
x = x.encode("utf-8")
return hashlib.sha1(x, usedforsecurity=False).hexdigest()
hash_utf8 = sha_utf8
elif _algorithm == "SHA-256":
def sha256_utf8(x: str | bytes) -> str:
if isinstance(x, str):
x = x.encode("utf-8")
return hashlib.sha256(x, usedforsecurity=False).hexdigest()
hash_utf8 = sha256_utf8
elif _algorithm == "SHA-512":
def sha512_utf8(x: str | bytes) -> str:
if isinstance(x, str):
x = x.encode("utf-8")
return hashlib.sha512(x, usedforsecurity=False).hexdigest()
hash_utf8 = sha512_utf8
if hash_utf8 is None:
return None
def KD(s: str, d: str) -> str:
return hash_utf8(f"{s}:{d}")
# XXX not implemented yet
entdig = None
p_parsed = urlparse(url)
#: path is request-uri defined in RFC 2616 which should not be empty
path = p_parsed.path or "/"
if p_parsed.query:
path += f"?{p_parsed.query}"
A1 = f"{self.username}:{realm}:{self.password}"
A2 = f"{method}:{path}"
HA1 = hash_utf8(A1)
HA2 = hash_utf8(A2)
if nonce == self._thread_local.last_nonce:
self._thread_local.nonce_count += 1
else:
self._thread_local.nonce_count = 1
ncvalue = f"{self._thread_local.nonce_count:08x}"
s = str(self._thread_local.nonce_count).encode("utf-8")
s += nonce.encode("utf-8")
s += time.ctime().encode("utf-8")
s += os.urandom(8)
cnonce = hashlib.sha1(s, usedforsecurity=False).hexdigest()[:16]
if _algorithm == "MD5-SESS":
HA1 = hash_utf8(f"{HA1}:{nonce}:{cnonce}") # type: ignore[reportConstantRedefinition] # RFC 2617 terminology
if not qop:
respdig = KD(HA1, f"{nonce}:{HA2}")
elif qop == "auth" or "auth" in qop.split(","):
noncebit = f"{nonce}:{ncvalue}:{cnonce}:auth:{HA2}"
respdig = KD(HA1, noncebit)
else:
# XXX handle auth-int.
return None
self._thread_local.last_nonce = nonce
# XXX should the partial digests be encoded too?
base = (
f'username="{self.username}", realm="{realm}", nonce="{nonce}", '
f'uri="{path}", response="{respdig}"'
)
if opaque:
base += f', opaque="{opaque}"'
if algorithm:
base += f', algorithm="{algorithm}"'
if entdig:
base += f', digest="{entdig}"'
if qop:
base += f', qop="auth", nc={ncvalue}, cnonce="{cnonce}"'
return f"Digest {base}"
def handle_redirect(self, r: Response, **kwargs: Any) -> None:
"""Reset num_401_calls counter on redirects."""
if r.is_redirect:
self._thread_local.num_401_calls = 1
def handle_401(self, r: Response, **kwargs: Any) -> Response:
"""
Takes the given response and tries digest-auth, if needed.
:rtype: requests.Response
"""
# If response is not 4xx, do not auth
# See https://github.com/psf/requests/issues/3772
if not 400 <= r.status_code < 500:
self._thread_local.num_401_calls = 1
return r
if self._thread_local.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
if (seek := getattr(r.request.body, "seek", None)) is not None:
seek(self._thread_local.pos)
s_auth = r.headers.get("www-authenticate", "")
if "digest" in s_auth.lower() and self._thread_local.num_401_calls < 2:
self._thread_local.num_401_calls += 1
pat = re.compile(r"digest ", flags=re.IGNORECASE)
self._thread_local.chal = parse_dict_header(pat.sub("", s_auth, count=1))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
cookie_jar = cast("CookieJar", prep._cookies) # type: ignore[reportPrivateUsage]
extract_cookies_to_jar(cookie_jar, r.request, r.raw)
prep.prepare_cookies(cookie_jar)
_digest_auth = self.build_digest_header(
cast(str, prep.method), cast(str, prep.url)
)
if _digest_auth:
prep.headers["Authorization"] = _digest_auth
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
self._thread_local.num_401_calls = 1
return r
def __call__(self, r: PreparedRequest) -> PreparedRequest:
# Initialize per-thread state, if needed
self.init_per_thread_state()
# If we have a saved nonce, skip the 401
if self._thread_local.last_nonce:
_digest_auth = self.build_digest_header(
cast(str, r.method), cast(str, r.url)
)
if _digest_auth:
r.headers["Authorization"] = _digest_auth
if (tell := getattr(r.body, "tell", None)) is not None:
self._thread_local.pos = tell()
else:
# In the case of HTTPDigestAuth being reused and the body of
# the previous request was a file-like object, pos has the
# file position of the previous body. Ensure it's set to
# None.
self._thread_local.pos = None
r.register_hook("response", self.handle_401)
r.register_hook("response", self.handle_redirect)
self._thread_local.num_401_calls = 1
return r
def __eq__(self, other: object) -> bool:
return all(
[
self.username == getattr(other, "username", None),
self.password == getattr(other, "password", None),
]
)
def __ne__(self, other: Any) -> bool:
return not self == other
@@ -0,0 +1,18 @@
#!/usr/bin/env python
"""
requests.certs
~~~~~~~~~~~~~~
This module returns the preferred default CA certificate bundle. There is
only one — the one from the certifi package.
If you are packaging Requests, e.g., for a Linux distribution or a managed
environment, you can change the definition of where() to return a separately
packaged CA bundle.
"""
from certifi import where
if __name__ == "__main__":
print(where())
@@ -0,0 +1,113 @@
"""
requests.compat
~~~~~~~~~~~~~~~
This module previously handled import compatibility issues
between Python 2 and Python 3. It remains for backwards
compatibility until the next major version.
"""
# pyright: reportUnusedImport=false
from __future__ import annotations
import importlib
import sys
from types import ModuleType
# -------
# urllib3
# -------
from urllib3 import (
__version__ as urllib3_version, # type: ignore[reportPrivateImportUsage]
)
# Detect which major version of urllib3 is being used.
try:
is_urllib3_1 = int(urllib3_version.split(".")[0]) == 1
except (TypeError, AttributeError):
# If we can't discern a version, prefer old functionality.
is_urllib3_1 = True
# -------------------
# Character Detection
# -------------------
def _resolve_char_detection() -> ModuleType | None:
"""Find supported character detection libraries."""
chardet = None
for lib in ("chardet", "charset_normalizer"):
if chardet is None:
try:
chardet = importlib.import_module(lib)
except ImportError:
pass
return chardet
chardet = _resolve_char_detection()
# -------
# Pythons
# -------
# Syntax sugar.
_ver = sys.version_info
#: Python 2.x?
is_py2 = _ver[0] == 2
#: Python 3.x?
is_py3 = _ver[0] == 3
# json/simplejson module import resolution
has_simplejson = False
try:
import simplejson as json # type: ignore[import-not-found]
has_simplejson = True
except ImportError:
import json
if has_simplejson:
from simplejson import JSONDecodeError # type: ignore[import-not-found]
else:
from json import JSONDecodeError
# Keep OrderedDict for backwards compatibility.
from collections import OrderedDict
from collections.abc import Callable, Mapping, MutableMapping
from http import cookiejar as cookielib
from http.cookies import Morsel
from io import StringIO
# --------------
# Legacy Imports
# --------------
from urllib.parse import (
quote,
quote_plus,
unquote,
unquote_plus,
urldefrag,
urlencode,
urljoin,
urlparse,
urlsplit,
urlunparse,
)
from urllib.request import (
getproxies,
getproxies_environment,
parse_http_list,
proxy_bypass,
proxy_bypass_environment, # type: ignore[attr-defined] # https://github.com/python/cpython/issues/145331
)
builtin_str = str
str = str
bytes = bytes
basestring = (str, bytes)
numeric_types = (int, float)
integer_types = (int,)
@@ -0,0 +1,625 @@
"""
requests.cookies
~~~~~~~~~~~~~~~~
Compatibility code to be able to use `http.cookiejar.CookieJar` with requests.
requests.utils imports from here, so be careful with imports.
"""
from __future__ import annotations
import calendar
import copy
import time
from collections.abc import Iterator, MutableMapping
from http.cookiejar import Cookie, CookieJar, CookiePolicy
from typing import TYPE_CHECKING, Any, TypeVar, overload
from ._internal_utils import to_native_string
from ._types import is_prepared as _is_prepared
from .compat import Morsel, cookielib, urlparse, urlunparse
if TYPE_CHECKING:
from _typeshed import SupportsKeysAndGetItem
from .models import PreparedRequest
import threading
class MockRequest:
"""Wraps a `requests.PreparedRequest` to mimic a `urllib2.Request`.
The code in `http.cookiejar.CookieJar` expects this interface in order to correctly
manage cookie policies, i.e., determine whether a cookie can be set, given the
domains of the request and the cookie.
The original request object is read-only. The client is responsible for collecting
the new headers via `get_new_headers()` and interpreting them appropriately. You
probably want `get_cookie_header`, defined below.
"""
type: str
def __init__(self, request: PreparedRequest) -> None:
assert _is_prepared(request)
self._r = request
self._new_headers: dict[str, str] = {}
self.type = urlparse(self._r.url).scheme
def get_type(self) -> str:
return self.type
def get_host(self) -> str:
return urlparse(self._r.url).netloc
def get_origin_req_host(self) -> str:
return self.get_host()
def get_full_url(self) -> str:
# Only return the response's URL if the user hadn't set the Host
# header
if not self._r.headers.get("Host"):
return self._r.url
# If they did set it, retrieve it and reconstruct the expected domain
host = to_native_string(self._r.headers["Host"], encoding="utf-8")
parsed = urlparse(self._r.url)
# Reconstruct the URL as we expect it
return urlunparse(
[
parsed.scheme,
host,
parsed.path,
parsed.params,
parsed.query,
parsed.fragment,
]
)
def is_unverifiable(self) -> bool:
return True
def has_header(self, name: str) -> bool:
return name in self._r.headers or name in self._new_headers
def get_header(self, name: str, default: str | None = None) -> str | None:
return self._r.headers.get(name, self._new_headers.get(name, default)) # type: ignore[return-value]
def add_header(self, key: str, val: str) -> None:
"""cookiejar has no legitimate use for this method; add it back if you find one."""
raise NotImplementedError(
"Cookie headers should be added with add_unredirected_header()"
)
def add_unredirected_header(self, name: str, value: str) -> None:
self._new_headers[name] = value
def get_new_headers(self) -> dict[str, str]:
return self._new_headers
@property
def unverifiable(self) -> bool:
return self.is_unverifiable()
@property
def origin_req_host(self) -> str:
return self.get_origin_req_host()
@property
def host(self) -> str:
return self.get_host()
class MockResponse:
"""Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`.
...what? Basically, expose the parsed HTTP headers from the server response
the way `http.cookiejar` expects to see them.
"""
def __init__(self, headers: Any) -> None:
"""Make a MockResponse for `cookiejar` to read.
:param headers: a httplib.HTTPMessage or analogous carrying the headers
"""
self._headers = headers
def info(self) -> Any:
return self._headers
def getheaders(self, name: str) -> Any:
self._headers.getheaders(name)
def extract_cookies_to_jar(
jar: CookieJar, request: PreparedRequest, response: Any
) -> None:
"""Extract the cookies from the response into a CookieJar.
:param jar: http.cookiejar.CookieJar (not necessarily a RequestsCookieJar)
:param request: our own requests.Request object
:param response: urllib3.HTTPResponse object
"""
if not (hasattr(response, "_original_response") and response._original_response):
return
# the _original_response field is the wrapped httplib.HTTPResponse object,
req = MockRequest(request)
# pull out the HTTPMessage with the headers and put it in the mock:
res = MockResponse(response._original_response.msg)
jar.extract_cookies(res, req) # type: ignore[arg-type]
def get_cookie_header(jar: CookieJar, request: PreparedRequest) -> str | None:
"""
Produce an appropriate Cookie header string to be sent with `request`, or None.
:rtype: str
"""
r = MockRequest(request)
jar.add_cookie_header(r) # type: ignore[arg-type]
return r.get_new_headers().get("Cookie")
def remove_cookie_by_name(
cookiejar: CookieJar, name: str, domain: str | None = None, path: str | None = None
) -> None:
"""Unsets a cookie by name, by default over all domains and paths.
Wraps CookieJar.clear(), is O(n).
"""
clearables: list[tuple[str, str, str]] = []
for cookie in cookiejar:
if cookie.name != name:
continue
if domain is not None and domain != cookie.domain:
continue
if path is not None and path != cookie.path:
continue
clearables.append((cookie.domain, cookie.path, cookie.name))
for domain, path, name in clearables:
cookiejar.clear(domain, path, name)
class CookieConflictError(RuntimeError):
"""There are two cookies that meet the criteria specified in the cookie jar.
Use .get and .set and include domain and path args in order to be more specific.
"""
class RequestsCookieJar(CookieJar, MutableMapping[str, str | None]): # type: ignore[misc]
"""Compatibility class; is a http.cookiejar.CookieJar, but exposes a dict
interface.
This is the CookieJar we create by default for requests and sessions that
don't specify one, since some clients may expect response.cookies and
session.cookies to support dict operations.
Requests does not use the dict interface internally; it's just for
compatibility with external client code. All requests code should work
out of the box with externally provided instances of ``CookieJar``, e.g.
``LWPCookieJar`` and ``FileCookieJar``.
Unlike a regular CookieJar, this class is pickleable.
.. warning:: dictionary operations that are normally O(1) may be O(n).
"""
_policy: CookiePolicy
def get( # type: ignore[override]
self,
name: str,
default: str | None = None,
domain: str | None = None,
path: str | None = None,
) -> str | None:
"""Dict-like get() that also supports optional domain and path args in
order to resolve naming collisions from using one cookie jar over
multiple domains.
.. warning:: operation is O(n), not O(1).
"""
try:
return self._find_no_duplicates(name, domain, path)
except KeyError:
return default
def set(
self, name: str, value: str | Morsel[dict[str, str]] | None, **kwargs: Any
) -> Cookie | None:
"""Dict-like set() that also supports optional domain and path args in
order to resolve naming collisions from using one cookie jar over
multiple domains.
"""
# support client code that unsets cookies by assignment of a None value:
if value is None:
remove_cookie_by_name(
self, name, domain=kwargs.get("domain"), path=kwargs.get("path")
)
return
if isinstance(value, Morsel):
c = morsel_to_cookie(value)
else:
c = create_cookie(name, value, **kwargs)
self.set_cookie(c)
return c
def iterkeys(self) -> Iterator[str]:
"""Dict-like iterkeys() that returns an iterator of names of cookies
from the jar.
.. seealso:: itervalues() and iteritems().
"""
for cookie in iter(self):
yield cookie.name
def keys(self) -> list[str]: # type: ignore[override]
"""Dict-like keys() that returns a list of names of cookies from the
jar.
.. seealso:: values() and items().
"""
return list(self.iterkeys())
def itervalues(self) -> Iterator[str | None]:
"""Dict-like itervalues() that returns an iterator of values of cookies
from the jar.
.. seealso:: iterkeys() and iteritems().
"""
for cookie in iter(self):
yield cookie.value
def values(self) -> list[str | None]: # type: ignore[override]
"""Dict-like values() that returns a list of values of cookies from the
jar.
.. seealso:: keys() and items().
"""
return list(self.itervalues())
def iteritems(self) -> Iterator[tuple[str, str | None]]:
"""Dict-like iteritems() that returns an iterator of name-value tuples
from the jar.
.. seealso:: iterkeys() and itervalues().
"""
for cookie in iter(self):
yield cookie.name, cookie.value
def items(self) -> list[tuple[str, str | None]]: # type: ignore[override]
"""Dict-like items() that returns a list of name-value tuples from the
jar. Allows client-code to call ``dict(RequestsCookieJar)`` and get a
vanilla python dict of key value pairs.
.. seealso:: keys() and values().
"""
return list(self.iteritems())
def list_domains(self) -> list[str]:
"""Utility method to list all the domains in the jar."""
domains: list[str] = []
for cookie in iter(self):
if cookie.domain not in domains:
domains.append(cookie.domain)
return domains
def list_paths(self) -> list[str]:
"""Utility method to list all the paths in the jar."""
paths: list[str] = []
for cookie in iter(self):
if cookie.path not in paths:
paths.append(cookie.path)
return paths
def multiple_domains(self) -> bool:
"""Returns True if there are multiple domains in the jar.
Returns False otherwise.
:rtype: bool
"""
domains: list[str] = []
for cookie in iter(self):
if cookie.domain is not None and cookie.domain in domains: # type: ignore[reportUnnecessaryComparison] # defensive check
return True
domains.append(cookie.domain)
return False # there is only one domain in jar
def get_dict(
self, domain: str | None = None, path: str | None = None
) -> dict[str, str | None]:
"""Takes as an argument an optional domain and path and returns a plain
old Python dict of name-value pairs of cookies that meet the
requirements.
:rtype: dict
"""
dictionary: dict[str, str | None] = {}
for cookie in iter(self):
if (domain is None or cookie.domain == domain) and (
path is None or cookie.path == path
):
dictionary[cookie.name] = cookie.value
return dictionary
def __iter__(self) -> Iterator[Cookie]: # type: ignore[override]
"""RequestCookieJar's __iter__ comes from CookieJar not MutableMapping."""
return super().__iter__()
def __contains__(self, name: object) -> bool:
try:
return super().__contains__(name)
except CookieConflictError:
return True
def __getitem__(self, name: str) -> str | None:
"""Dict-like __getitem__() for compatibility with client code. Throws
exception if there are more than one cookie with name. In that case,
use the more explicit get() method instead.
.. warning:: operation is O(n), not O(1).
"""
return self._find_no_duplicates(name)
def __setitem__(
self, name: str, value: str | Morsel[dict[str, str]] | None
) -> None:
"""Dict-like __setitem__ for compatibility with client code. Throws
exception if there is already a cookie of that name in the jar. In that
case, use the more explicit set() method instead.
"""
self.set(name, value)
def __delitem__(self, name: str) -> None:
"""Deletes a cookie given a name. Wraps ``http.cookiejar.CookieJar``'s
``remove_cookie_by_name()``.
"""
remove_cookie_by_name(self, name)
def set_cookie(self, cookie: Cookie, *args: Any, **kwargs: Any) -> None:
if (
(value := cookie.value) is not None
and value.startswith('"')
and value.endswith('"')
):
cookie.value = value.replace('\\"', "")
return super().set_cookie(cookie, *args, **kwargs)
def update( # type: ignore[override]
self, other: CookieJar | SupportsKeysAndGetItem[str, str]
) -> None:
"""Updates this jar with cookies from another CookieJar or dict-like"""
if isinstance(other, cookielib.CookieJar):
for cookie in other:
self.set_cookie(copy.copy(cookie))
else:
super().update(other)
def _find(
self, name: str, domain: str | None = None, path: str | None = None
) -> str | None:
"""Requests uses this method internally to get cookie values.
If there are conflicting cookies, _find arbitrarily chooses one.
See _find_no_duplicates if you want an exception thrown if there are
conflicting cookies.
:param name: a string containing name of cookie
:param domain: (optional) string containing domain of cookie
:param path: (optional) string containing path of cookie
:return: cookie.value
"""
for cookie in iter(self):
if cookie.name == name:
if domain is None or cookie.domain == domain:
if path is None or cookie.path == path:
return cookie.value
raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}")
def _find_no_duplicates(
self, name: str, domain: str | None = None, path: str | None = None
) -> str:
"""Both ``__get_item__`` and ``get`` call this function: it's never
used elsewhere in Requests.
:param name: a string containing name of cookie
:param domain: (optional) string containing domain of cookie
:param path: (optional) string containing path of cookie
:raises KeyError: if cookie is not found
:raises CookieConflictError: if there are multiple cookies
that match name and optionally domain and path
:return: cookie.value
"""
toReturn = None
for cookie in iter(self):
if cookie.name == name:
if domain is None or cookie.domain == domain:
if path is None or cookie.path == path:
if toReturn is not None:
# if there are multiple cookies that meet passed in criteria
raise CookieConflictError(
f"There are multiple cookies with name, {name!r}"
)
# we will eventually return this as long as no cookie conflict
toReturn = cookie.value
if toReturn is not None:
return toReturn
raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}")
def __getstate__(self) -> dict[str, Any]:
"""Unlike a normal CookieJar, this class is pickleable."""
state = self.__dict__.copy()
# remove the unpickleable RLock object
state.pop("_cookies_lock")
return state
def __setstate__(self, state: dict[str, Any]) -> None:
"""Unlike a normal CookieJar, this class is pickleable."""
self.__dict__.update(state)
if "_cookies_lock" not in self.__dict__:
self._cookies_lock = threading.RLock()
def copy(self) -> RequestsCookieJar:
"""Return a copy of this RequestsCookieJar."""
new_cj = RequestsCookieJar()
new_cj.set_policy(self.get_policy())
new_cj.update(self)
return new_cj
def get_policy(self) -> CookiePolicy:
"""Return the CookiePolicy instance used."""
return self._policy
def _copy_cookie_jar(jar: CookieJar | None) -> CookieJar | None: # type: ignore[reportUnusedFunction] # cross-module usage in models.py
if jar is None:
return None
if copy_method := getattr(jar, "copy", None):
# We're dealing with an instance of RequestsCookieJar
return copy_method()
# We're dealing with a generic CookieJar instance
new_jar = copy.copy(jar)
new_jar.clear()
for cookie in jar:
new_jar.set_cookie(copy.copy(cookie))
return new_jar
def create_cookie(name: str, value: str, **kwargs: Any) -> Cookie:
"""Make a cookie from underspecified parameters.
By default, the pair of `name` and `value` will be set for the domain ''
and sent on every request (this is sometimes called a "supercookie").
"""
result: dict[str, Any] = {
"version": 0,
"name": name,
"value": value,
"port": None,
"domain": "",
"path": "/",
"secure": False,
"expires": None,
"discard": True,
"comment": None,
"comment_url": None,
"rest": {"HttpOnly": None},
"rfc2109": False,
}
badargs = set(kwargs) - set(result)
if badargs:
raise TypeError(
f"create_cookie() got unexpected keyword arguments: {list(badargs)}"
)
result.update(kwargs)
result["port_specified"] = bool(result["port"])
result["domain_specified"] = bool(result["domain"])
result["domain_initial_dot"] = result["domain"].startswith(".")
result["path_specified"] = bool(result["path"])
return cookielib.Cookie(**result)
def morsel_to_cookie(morsel: Morsel[Any]) -> Cookie:
"""Convert a Morsel object into a Cookie containing the one k/v pair."""
expires: int | None = None
if morsel["max-age"]:
try:
expires = int(time.time() + int(morsel["max-age"]))
except ValueError:
raise TypeError(f"max-age: {morsel['max-age']} must be integer")
elif morsel["expires"]:
time_template = "%a, %d-%b-%Y %H:%M:%S GMT"
expires = calendar.timegm(time.strptime(morsel["expires"], time_template))
return create_cookie(
comment=morsel["comment"],
comment_url=bool(morsel["comment"]),
discard=False,
domain=morsel["domain"],
expires=expires,
name=morsel.key,
path=morsel["path"],
port=None,
rest={"HttpOnly": morsel["httponly"]},
rfc2109=False,
secure=bool(morsel["secure"]),
value=morsel.value,
version=morsel["version"] or 0,
)
_CookieJarT = TypeVar("_CookieJarT", bound=CookieJar)
@overload
def cookiejar_from_dict(
cookie_dict: dict[str, str] | None,
cookiejar: None = None,
overwrite: bool = True,
) -> RequestsCookieJar: ...
@overload
def cookiejar_from_dict(
cookie_dict: dict[str, str] | None,
cookiejar: _CookieJarT,
overwrite: bool = True,
) -> _CookieJarT: ...
def cookiejar_from_dict(
cookie_dict: dict[str, str] | None,
cookiejar: CookieJar | None = None,
overwrite: bool = True,
) -> CookieJar:
"""Returns a CookieJar from a key/value dictionary.
:param cookie_dict: Dict of key/values to insert into CookieJar.
:param cookiejar: (optional) A cookiejar to add the cookies to.
:param overwrite: (optional) If False, will not replace cookies
already in the jar with new ones.
:rtype: CookieJar
"""
if cookiejar is None:
cookiejar = RequestsCookieJar()
if cookie_dict is not None:
names_from_jar = [cookie.name for cookie in cookiejar]
for name in cookie_dict:
if overwrite or (name not in names_from_jar):
cookiejar.set_cookie(create_cookie(name, cookie_dict[name]))
return cookiejar
def merge_cookies(
cookiejar: CookieJar, cookies: dict[str, str] | CookieJar | None
) -> CookieJar:
"""Add cookies to cookiejar and returns a merged CookieJar.
:param cookiejar: CookieJar object to add the cookies to.
:param cookies: Dictionary or CookieJar object to be added.
:rtype: CookieJar
"""
if not isinstance(cookiejar, cookielib.CookieJar): # type: ignore[reportUnnecessaryIsInstance] # runtime guard
raise ValueError("You can only merge into CookieJar")
if isinstance(cookies, dict):
cookiejar = cookiejar_from_dict(cookies, cookiejar=cookiejar, overwrite=False)
elif isinstance(cookies, cookielib.CookieJar):
if update_method := getattr(cookiejar, "update", None):
update_method(cookies)
else:
for cookie_in_jar in cookies:
cookiejar.set_cookie(cookie_in_jar)
return cookiejar
@@ -0,0 +1,162 @@
"""
requests.exceptions
~~~~~~~~~~~~~~~~~~~
This module contains the set of Requests' exceptions.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from urllib3.exceptions import HTTPError as BaseHTTPError
from .compat import JSONDecodeError as CompatJSONDecodeError
if TYPE_CHECKING:
from .models import PreparedRequest, Request, Response
class RequestException(IOError):
"""There was an ambiguous exception that occurred while handling your
request.
"""
response: Response | None
request: Request | PreparedRequest | None
def __init__(self, *args: Any, **kwargs: Any) -> None:
"""Initialize RequestException with `request` and `response` objects."""
response: Response | None = kwargs.pop("response", None)
self.response = response
self.request = kwargs.pop("request", None)
if response is not None and not self.request and hasattr(response, "request"):
self.request = response.request
super().__init__(*args, **kwargs)
class InvalidJSONError(RequestException):
"""A JSON error occurred."""
class JSONDecodeError(InvalidJSONError, CompatJSONDecodeError):
"""Couldn't decode the text into json"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
"""
Construct the JSONDecodeError instance first with all
args. Then use it's args to construct the IOError so that
the json specific args aren't used as IOError specific args
and the error message from JSONDecodeError is preserved.
"""
CompatJSONDecodeError.__init__(self, *args)
InvalidJSONError.__init__(self, *self.args, **kwargs)
def __reduce__(self) -> tuple[Any, ...] | str:
"""
The __reduce__ method called when pickling the object must
be the one from the JSONDecodeError (be it json/simplejson)
as it expects all the arguments for instantiation, not just
one like the IOError, and the MRO would by default call the
__reduce__ method from the IOError due to the inheritance order.
"""
return CompatJSONDecodeError.__reduce__(self)
class HTTPError(RequestException):
"""An HTTP error occurred."""
class ConnectionError(RequestException):
"""A Connection error occurred."""
class ProxyError(ConnectionError):
"""A proxy error occurred."""
class SSLError(ConnectionError):
"""An SSL error occurred."""
class Timeout(RequestException):
"""The request timed out.
Catching this error will catch both
:exc:`~requests.exceptions.ConnectTimeout` and
:exc:`~requests.exceptions.ReadTimeout` errors.
"""
class ConnectTimeout(ConnectionError, Timeout):
"""The request timed out while trying to connect to the remote server.
Requests that produced this error are safe to retry.
"""
class ReadTimeout(Timeout):
"""The server did not send any data in the allotted amount of time."""
class URLRequired(RequestException):
"""A valid URL is required to make a request."""
class TooManyRedirects(RequestException):
"""Too many redirects."""
class MissingSchema(RequestException, ValueError):
"""The URL scheme (e.g. http or https) is missing."""
class InvalidSchema(RequestException, ValueError):
"""The URL scheme provided is either invalid or unsupported."""
class InvalidURL(RequestException, ValueError):
"""The URL provided was somehow invalid."""
class InvalidHeader(RequestException, ValueError):
"""The header value provided was somehow invalid."""
class InvalidProxyURL(InvalidURL):
"""The proxy URL provided is invalid."""
class ChunkedEncodingError(RequestException):
"""The server declared chunked encoding but sent an invalid chunk."""
class ContentDecodingError(RequestException, BaseHTTPError):
"""Failed to decode response content."""
class StreamConsumedError(RequestException, TypeError):
"""The content for this response was already consumed."""
class RetryError(RequestException):
"""Custom retries logic failed"""
class UnrewindableBodyError(RequestException):
"""Requests encountered an error when trying to rewind a body."""
# Warnings
class RequestsWarning(Warning):
"""Base warning for Requests."""
class FileModeWarning(RequestsWarning, DeprecationWarning):
"""A file was opened in text mode, but Requests determined its binary length."""
class RequestsDependencyWarning(RequestsWarning):
"""An imported dependency doesn't match the expected version range."""
@@ -0,0 +1,134 @@
"""Module containing bug report helper(s)."""
# pyright: reportUnknownMemberType=false
import json
import platform
import ssl
import sys
from typing import Any
import idna
import urllib3
from . import __version__ as requests_version
try:
import charset_normalizer
except ImportError:
charset_normalizer = None
try:
import chardet # type: ignore[import-not-found]
except ImportError:
chardet = None
try:
from urllib3.contrib import pyopenssl
except ImportError:
pyopenssl = None
OpenSSL = None
cryptography = None
else:
import cryptography # type: ignore[import-not-found]
import OpenSSL # type: ignore[import-not-found]
def _implementation():
"""Return a dict with the Python implementation and version.
Provide both the name and the version of the Python implementation
currently running. For example, on CPython 3.10.3 it will return
{'name': 'CPython', 'version': '3.10.3'}.
This function works best on CPython and PyPy: in particular, it probably
doesn't work for Jython or IronPython. Future investigation should be done
to work out the correct shape of the code for those platforms.
"""
implementation = platform.python_implementation()
if implementation == "CPython":
implementation_version = platform.python_version()
elif implementation == "PyPy":
pypy = sys.pypy_version_info # type: ignore[attr-defined]
implementation_version = f"{pypy.major}.{pypy.minor}.{pypy.micro}"
if sys.pypy_version_info.releaselevel != "final": # type: ignore[attr-defined]
implementation_version = "".join(
[implementation_version, sys.pypy_version_info.releaselevel] # type: ignore[attr-defined]
)
elif implementation == "Jython":
implementation_version = platform.python_version() # Complete Guess
elif implementation == "IronPython":
implementation_version = platform.python_version() # Complete Guess
else:
implementation_version = "Unknown"
return {"name": implementation, "version": implementation_version}
def info() -> dict[str, Any]:
"""Generate information for a bug report."""
try:
platform_info = {
"system": platform.system(),
"release": platform.release(),
}
except OSError:
platform_info = {
"system": "Unknown",
"release": "Unknown",
}
implementation_info = _implementation()
urllib3_info = {"version": urllib3.__version__} # type: ignore[reportPrivateImportUsage]
charset_normalizer_info = {"version": None}
chardet_info: dict[str, str | None] = {"version": None}
if charset_normalizer:
charset_normalizer_info = {"version": charset_normalizer.__version__}
if chardet:
chardet_info = {"version": chardet.__version__}
pyopenssl_info: dict[str, str | None] = {
"version": None,
"openssl_version": "",
}
if OpenSSL:
pyopenssl_info = {
"version": OpenSSL.__version__,
"openssl_version": f"{OpenSSL.SSL.OPENSSL_VERSION_NUMBER:x}",
}
cryptography_info = {
"version": getattr(cryptography, "__version__", ""),
}
idna_info = {
"version": getattr(idna, "__version__", ""),
}
system_ssl = ssl.OPENSSL_VERSION_NUMBER
system_ssl_info = {"version": f"{system_ssl:x}" if system_ssl is not None else ""} # type: ignore[reportUnnecessaryComparison]
return {
"platform": platform_info,
"implementation": implementation_info,
"system_ssl": system_ssl_info,
"using_pyopenssl": pyopenssl is not None,
"using_charset_normalizer": chardet is None,
"pyOpenSSL": pyopenssl_info,
"urllib3": urllib3_info,
"chardet": chardet_info,
"charset_normalizer": charset_normalizer_info,
"cryptography": cryptography_info,
"idna": idna_info,
"requests": {
"version": requests_version,
},
}
def main():
"""Pretty-print the bug information as JSON."""
print(json.dumps(info(), sort_keys=True, indent=2))
if __name__ == "__main__":
main()
@@ -0,0 +1,48 @@
"""
requests.hooks
~~~~~~~~~~~~~~
This module provides the capabilities for the Requests hooks system.
Available hooks:
``response``:
The response generated from a Request.
"""
from __future__ import annotations
from collections.abc import Callable, Iterable
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from . import _types as _t
from .models import Response
HOOKS: list[str] = ["response"]
def default_hooks() -> dict[str, list[_t.HookType]]:
return {event: [] for event in HOOKS}
# TODO: response is the only one
def dispatch_hook(
key: str,
hooks: _t.HooksInputType | None,
hook_data: Response,
**kwargs: Any,
) -> Response:
"""Dispatches a hook dictionary on a given piece of data."""
hooks_dict = hooks or {}
hook_list: Iterable[_t.HookType] | _t.HookType | None = hooks_dict.get(key)
if hook_list:
if isinstance(hook_list, Callable):
hook_list = [hook_list]
for hook in hook_list:
_hook_data = hook(hook_data, **kwargs)
if _hook_data is not None:
hook_data = _hook_data
return hook_data
@@ -0,0 +1,23 @@
import sys
from .compat import chardet
# This code exists for backwards compatibility reasons.
# I don't like it either. Just look the other way. :)
for package in ("urllib3", "idna"):
locals()[package] = __import__(package)
# This traversal is apparently necessary such that the identities are
# preserved (requests.packages.urllib3.* is urllib3.*)
for mod in list(sys.modules):
if mod == package or mod.startswith(f"{package}."):
sys.modules[f"requests.packages.{mod}"] = sys.modules[mod]
if chardet is not None:
target = chardet.__name__
for mod in list(sys.modules):
if mod == target or mod.startswith(f"{target}."):
imported_mod = sys.modules[mod]
sys.modules[f"requests.packages.{mod}"] = imported_mod
mod = mod.replace(target, "chardet")
sys.modules[f"requests.packages.{mod}"] = imported_mod
@@ -0,0 +1,920 @@
"""
requests.sessions
~~~~~~~~~~~~~~~~~
This module provides a Session object to manage and persist settings across
requests (cookies, auth, proxies).
"""
from __future__ import annotations
import os
import sys
import time
from collections import OrderedDict
from collections.abc import Generator, Mapping, MutableMapping
from datetime import timedelta
from typing import TYPE_CHECKING, Any, cast
from ._internal_utils import to_native_string
from ._types import is_prepared as _is_prepared
from .adapters import HTTPAdapter
from .auth import _basic_auth_str # type: ignore[reportPrivateUsage]
from .compat import cookielib, urljoin, urlparse
from .cookies import (
RequestsCookieJar,
cookiejar_from_dict,
extract_cookies_to_jar,
merge_cookies,
)
from .exceptions import (
ChunkedEncodingError,
ContentDecodingError,
InvalidSchema,
TooManyRedirects,
)
from .hooks import default_hooks, dispatch_hook
# formerly defined here, reexposed here for backward compatibility
from .models import ( # noqa: F401
DEFAULT_REDIRECT_LIMIT,
REDIRECT_STATI, # type: ignore[reportUnusedImport]
PreparedRequest,
Request,
Response,
)
from .status_codes import codes
from .structures import CaseInsensitiveDict
from .utils import ( # noqa: F401
DEFAULT_PORTS,
default_headers,
get_auth_from_url,
get_environ_proxies,
get_netrc_auth,
requote_uri,
resolve_proxies,
rewind_body,
should_bypass_proxies, # type: ignore[reportUnusedImport] # re-export for external consumers
to_key_val_list,
)
if TYPE_CHECKING:
from http.cookiejar import CookieJar
from typing_extensions import Self, Unpack
from . import _types as _t
from .adapters import BaseAdapter
# Preferred clock, based on which one is more accurate on a given system.
if sys.platform == "win32":
preferred_clock = time.perf_counter
else:
preferred_clock = time.time
def merge_setting(
request_setting: Any, session_setting: Any, dict_class: type = OrderedDict
) -> Any:
"""Determines appropriate setting for a given request, taking into account
the explicit setting on that request, and the setting in the session. If a
setting is a dictionary, they will be merged together using `dict_class`
"""
if session_setting is None:
return request_setting
if request_setting is None:
return session_setting
# Bypass if not a dictionary (e.g. verify)
if not (
isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping)
):
return request_setting
merged_setting = dict_class(to_key_val_list(session_setting)) # type: ignore[arg-type] # isinstance narrows Any to Mapping[Unknown]
merged_setting.update(to_key_val_list(request_setting)) # type: ignore[arg-type]
# Remove keys that are set to None. Extract keys first to avoid altering
# the dictionary during iteration.
none_keys = [k for (k, v) in merged_setting.items() if v is None]
for key in none_keys:
del merged_setting[key]
return merged_setting
def merge_hooks(
request_hooks: _t.HooksType,
session_hooks: _t.HooksType,
dict_class: type = OrderedDict,
) -> _t.HooksType:
"""Properly merges both requests and session hooks.
This is necessary because when request_hooks == {'response': []}, the
merge breaks Session hooks entirely.
"""
if session_hooks is None or session_hooks.get("response") == []:
return request_hooks
if request_hooks is None or request_hooks.get("response") == []:
return session_hooks
return merge_setting(request_hooks, session_hooks, dict_class)
class SessionRedirectMixin:
max_redirects: int
trust_env: bool
cookies: RequestsCookieJar
def send(self, request: PreparedRequest, **kwargs: Any) -> Response: ...
def get_redirect_target(self, resp: Response) -> str | None:
"""Receives a Response. Returns a redirect URI or ``None``"""
# Due to the nature of how requests processes redirects this method will
# be called at least once upon the original response and at least twice
# on each subsequent redirect response (if any).
# If a custom mixin is used to handle this logic, it may be advantageous
# to cache the redirect location onto the response object as a private
# attribute.
if resp.is_redirect:
location = resp.headers["location"]
# Currently the underlying http module on py3 decode headers
# in latin1, but empirical evidence suggests that latin1 is very
# rarely used with non-ASCII characters in HTTP headers.
# It is more likely to get UTF8 header rather than latin1.
# This causes incorrect handling of UTF8 encoded location headers.
# To solve this, we re-encode the location in latin1.
location = location.encode("latin1")
return to_native_string(location, "utf8")
return None
def should_strip_auth(self, old_url: str, new_url: str) -> bool:
"""Decide whether Authorization header should be removed when redirecting"""
old_parsed = urlparse(old_url)
new_parsed = urlparse(new_url)
if old_parsed.hostname != new_parsed.hostname:
return True
# Special case: allow http -> https redirect when using the standard
# ports. This isn't specified by RFC 7235, but is kept to avoid
# breaking backwards compatibility with older versions of requests
# that allowed any redirects on the same host.
if (
old_parsed.scheme == "http"
and old_parsed.port in (80, None)
and new_parsed.scheme == "https"
and new_parsed.port in (443, None)
):
return False
# Handle default port usage corresponding to scheme.
changed_port = old_parsed.port != new_parsed.port
changed_scheme = old_parsed.scheme != new_parsed.scheme
default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None)
if (
not changed_scheme
and old_parsed.port in default_port
and new_parsed.port in default_port
):
return False
# Standard case: root URI must match
return changed_port or changed_scheme
def resolve_redirects(
self,
resp: Response,
req: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
yield_requests: bool = False,
**adapter_kwargs: Any,
) -> Generator[Response, None, None]:
"""Receives a Response. Returns a generator of Responses or Requests."""
hist: list[Response] = [] # keep track of history
url = self.get_redirect_target(resp)
previous_fragment = urlparse(req.url).fragment
while url:
prepared_request = req.copy()
# Update history and keep track of redirects.
resp.history = hist[:]
hist.append(resp)
try:
resp.content # Consume socket so it can be released
except (ChunkedEncodingError, ContentDecodingError, RuntimeError):
resp.raw.read(decode_content=False)
if len(resp.history) >= self.max_redirects:
raise TooManyRedirects(
f"Exceeded {self.max_redirects} redirects.", response=resp
)
# Release the connection back into the pool.
resp.close()
# Handle redirection without scheme (see: RFC 1808 Section 4)
if url.startswith("//"):
parsed_rurl = urlparse(resp.url)
url = ":".join([to_native_string(parsed_rurl.scheme), url])
# Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2)
parsed = urlparse(url)
if parsed.fragment == "" and previous_fragment:
parsed = parsed._replace(fragment=previous_fragment)
elif parsed.fragment:
previous_fragment = parsed.fragment
url = parsed.geturl()
# Facilitate relative 'location' headers, as allowed by RFC 7231.
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
# Compliant with RFC3986, we percent encode the url.
if not parsed.netloc:
url = urljoin(resp.url, requote_uri(url))
else:
url = requote_uri(url)
prepared_request.url = to_native_string(url)
self.rebuild_method(prepared_request, resp)
# https://github.com/psf/requests/issues/1084
if resp.status_code not in (
codes.temporary_redirect,
codes.permanent_redirect,
):
# https://github.com/psf/requests/issues/3490
purged_headers = ("Content-Length", "Content-Type", "Transfer-Encoding")
for header in purged_headers:
prepared_request.headers.pop(header, None)
prepared_request.body = None
headers = prepared_request.headers
headers.pop("Cookie", None)
# Extract any cookies sent on the response to the cookiejar
# in the new request. Because we've mutated our copied prepared
# request, use the old one that we haven't yet touched.
cookie_jar = cast("CookieJar", prepared_request._cookies) # type: ignore[reportPrivateUsage]
extract_cookies_to_jar(cookie_jar, req, resp.raw)
merge_cookies(cookie_jar, self.cookies)
prepared_request.prepare_cookies(cookie_jar)
# Rebuild auth and proxy information.
proxies = self.rebuild_proxies(prepared_request, proxies)
self.rebuild_auth(prepared_request, resp)
# A failed tell() sets `_body_position` to `object()`. This non-None
# value ensures `rewindable` will be True, allowing us to raise an
# UnrewindableBodyError, instead of hanging the connection.
rewindable = prepared_request._body_position is not None and ( # type: ignore[reportPrivateUsage]
"Content-Length" in headers or "Transfer-Encoding" in headers
)
# Attempt to rewind consumed file-like object.
if rewindable:
rewind_body(prepared_request)
# Override the original request.
req = prepared_request
if yield_requests:
yield req # type: ignore[misc] # Internal use only, returns PreparedRequest
else:
resp = self.send(
req,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
allow_redirects=False,
**adapter_kwargs,
)
extract_cookies_to_jar(self.cookies, prepared_request, resp.raw)
# extract redirect url, if any, for the next loop
url = self.get_redirect_target(resp)
yield resp
def rebuild_auth(
self, prepared_request: PreparedRequest, response: Response
) -> None:
"""When being redirected we may want to strip authentication from the
request to avoid leaking credentials. This method intelligently removes
and reapplies authentication where possible to avoid credential loss.
"""
original_request = response.request
assert _is_prepared(original_request)
assert _is_prepared(prepared_request)
headers = prepared_request.headers
original_url = original_request.url
url = prepared_request.url
if "Authorization" in headers and self.should_strip_auth(original_url, url):
# If we get redirected to a new host, we should strip out any
# authentication headers.
del headers["Authorization"]
# .netrc might have more auth for us on our new host.
new_auth = get_netrc_auth(url) if self.trust_env else None
if new_auth is not None:
prepared_request.prepare_auth(new_auth)
def rebuild_proxies(
self,
prepared_request: PreparedRequest,
proxies: dict[str, str] | None,
) -> dict[str, str]:
"""This method re-evaluates the proxy configuration by considering the
environment variables. If we are redirected to a URL covered by
NO_PROXY, we strip the proxy configuration. Otherwise, we set missing
proxy keys for this URL (in case they were stripped by a previous
redirect).
This method also replaces the Proxy-Authorization header where
necessary.
:rtype: dict
"""
assert _is_prepared(prepared_request)
headers = prepared_request.headers
scheme = urlparse(prepared_request.url).scheme
new_proxies = resolve_proxies(prepared_request, proxies, self.trust_env)
if "Proxy-Authorization" in headers:
del headers["Proxy-Authorization"]
try:
username, password = get_auth_from_url(new_proxies[scheme])
except KeyError:
username, password = None, None
# urllib3 handles proxy authorization for us in the standard adapter.
# Avoid appending this to TLS tunneled requests where it may be leaked.
if not scheme.startswith("https") and username and password:
headers["Proxy-Authorization"] = _basic_auth_str(username, password)
return new_proxies
def rebuild_method(
self, prepared_request: PreparedRequest, response: Response
) -> None:
"""When being redirected we may want to change the method of the request
based on certain specs or browser behavior.
"""
method = prepared_request.method
# https://tools.ietf.org/html/rfc7231#section-6.4.4
if response.status_code == codes.see_other and method != "HEAD":
method = "GET"
# Do what the browsers do, despite standards...
# First, turn 302s into GETs.
if response.status_code == codes.found and method != "HEAD":
method = "GET"
# Second, if a POST is responded to with a 301, turn it into a GET.
# This bizarre behaviour is explained in Issue 1704.
if response.status_code == codes.moved and method == "POST":
method = "GET"
prepared_request.method = method
class Session(SessionRedirectMixin):
"""A Requests session.
Provides cookie persistence, connection-pooling, and configuration.
Basic Usage::
>>> import requests
>>> s = requests.Session()
>>> s.get('https://httpbin.org/get')
<Response [200]>
Or as a context manager::
>>> with requests.Session() as s:
... s.get('https://httpbin.org/get')
<Response [200]>
"""
headers: CaseInsensitiveDict[str]
auth: _t.AuthType
proxies: dict[str, str]
hooks: dict[str, list[_t.HookType]]
params: MutableMapping[str, Any]
stream: bool
verify: _t.VerifyType
cert: _t.CertType
max_redirects: int
trust_env: bool
cookies: RequestsCookieJar
adapters: MutableMapping[str, BaseAdapter]
__attrs__: list[str] = [
"headers",
"cookies",
"auth",
"proxies",
"hooks",
"params",
"verify",
"cert",
"adapters",
"stream",
"trust_env",
"max_redirects",
]
def __init__(self) -> None:
#: A case-insensitive dictionary of headers to be sent on each
#: :class:`Request <Request>` sent from this
#: :class:`Session <Session>`.
self.headers = default_headers()
#: Default Authentication tuple or object to attach to
#: :class:`Request <Request>`.
self.auth = None
#: Dictionary mapping protocol or protocol and host to the URL of the proxy
#: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to
#: be used on each :class:`Request <Request>`.
self.proxies = {}
#: Event-handling hooks.
self.hooks = default_hooks()
#: Dictionary of querystring data to attach to each
#: :class:`Request <Request>`. The dictionary values may be lists for
#: representing multivalued query parameters.
self.params = {}
#: Stream response content default.
self.stream = False
#: SSL Verification default.
#: Defaults to `True`, requiring requests to verify the TLS certificate at the
#: remote end.
#: If verify is set to `False`, requests will accept any TLS certificate
#: presented by the server, and will ignore hostname mismatches and/or
#: expired certificates, which will make your application vulnerable to
#: man-in-the-middle (MitM) attacks.
#: Only set this to `False` for testing.
#: If verify is set to a string, it must be the path to a CA bundle file
#: that will be used to verify the TLS certificate.
self.verify = True
#: SSL client certificate default, if String, path to ssl client
#: cert file (.pem). If Tuple, ('cert', 'key') pair.
self.cert = None
#: Maximum number of redirects allowed. If the request exceeds this
#: limit, a :class:`TooManyRedirects` exception is raised.
#: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is
#: 30.
self.max_redirects = DEFAULT_REDIRECT_LIMIT
#: Trust environment settings for proxy configuration, default
#: authentication and similar.
self.trust_env = True
#: A CookieJar containing all currently outstanding cookies set on this
#: session. By default it is a
#: :class:`RequestsCookieJar <requests.cookies.RequestsCookieJar>`, but
#: may be any other ``cookielib.CookieJar`` compatible object.
self.cookies = cookiejar_from_dict({})
# Default connection adapters.
self.adapters = OrderedDict()
self.mount("https://", HTTPAdapter())
self.mount("http://", HTTPAdapter())
def __enter__(self) -> Self:
return self
def __exit__(self, *args: Any) -> None:
self.close()
def prepare_request(self, request: Request) -> PreparedRequest:
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for
transmission and returns it. The :class:`PreparedRequest` has settings
merged from the :class:`Request <Request>` instance and those of the
:class:`Session`.
:param request: :class:`Request` instance to prepare with this
session's settings.
:rtype: requests.PreparedRequest
"""
url = cast("_t.UriType", request.url)
method = cast(str, request.method)
cookies = request.cookies or {}
# Bootstrap CookieJar.
if not isinstance(cookies, cookielib.CookieJar):
cookies = cookiejar_from_dict(cookies)
# Merge with session cookies
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies
)
# Set environment's basic authentication if not explicitly set.
auth = request.auth
if self.trust_env and not auth and not self.auth:
auth = get_netrc_auth(url)
p = PreparedRequest()
p.prepare(
method=method.upper(),
url=url,
files=request.files,
data=request.data,
json=request.json,
headers=merge_setting(
request.headers, self.headers, dict_class=CaseInsensitiveDict
),
params=merge_setting(request.params, self.params),
auth=merge_setting(auth, self.auth),
cookies=merged_cookies,
hooks=merge_hooks(request.hooks, self.hooks),
)
return p
def request(
self,
method: str,
url: _t.UriType,
params: _t.ParamsType = None,
data: _t.DataType = None,
headers: _t.HeadersType = None,
cookies: RequestsCookieJar | CookieJar | dict[str, str] | None = None,
files: _t.FilesType = None,
auth: _t.AuthType = None,
timeout: _t.TimeoutType = None,
allow_redirects: bool = True,
proxies: dict[str, str] | None = None,
hooks: _t.HooksInputType | None = None,
stream: bool | None = None,
verify: _t.VerifyType | None = None,
cert: _t.CertType = None,
json: _t.JsonType = None,
) -> Response:
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) json to send in the body of the
:class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the
:class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the
:class:`Request`.
:param files: (optional) Dictionary of ``'filename': file-like-objects``
for multipart encoding upload.
:param auth: (optional) Auth tuple or callable to enable
Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) How many seconds to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param allow_redirects: (optional) Set to True by default.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol or protocol and
hostname to the URL of the proxy.
:param hooks: (optional) Dictionary mapping hook name to one event or
list of events, event must be callable.
:param stream: (optional) whether to immediately download the response
content. Defaults to ``False``.
:param verify: (optional) Either a boolean, in which case it controls whether we verify
the server's TLS certificate, or a string, in which case it must be a path
to a CA bundle to use. Defaults to ``True``. When set to
``False``, requests will accept any TLS certificate presented by
the server, and will ignore hostname mismatches and/or expired
certificates, which will make your application vulnerable to
man-in-the-middle (MitM) attacks. Setting verify to ``False``
may be useful during local development or testing.
:param cert: (optional) if String, path to ssl client cert file (.pem).
If Tuple, ('cert', 'key') pair.
:rtype: requests.Response
"""
if isinstance(url, bytes):
url = url.decode("utf-8")
# Create the Request.
req = Request(
method=method.upper(),
url=url,
headers=headers,
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
assert _is_prepared(prep)
proxies = proxies or {}
settings = self.merge_environment_settings(
prep.url, proxies, stream, verify, cert
)
# Send the request.
send_kwargs = {
"timeout": timeout,
"allow_redirects": allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
return resp
def get(
self,
url: _t.UriType,
params: _t.ParamsType = None,
**kwargs: Unpack[_t.GetKwargs],
) -> Response:
r"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary, list of tuples or bytes to send
in the query string for the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", True)
return self.request("GET", url, params=params, **kwargs)
def options(self, url: _t.UriType, **kwargs: Unpack[_t.RequestKwargs]) -> Response:
r"""Sends a OPTIONS request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", True)
return self.request("OPTIONS", url, **kwargs)
def head(self, url: _t.UriType, **kwargs: Unpack[_t.RequestKwargs]) -> Response:
r"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", False)
return self.request("HEAD", url, **kwargs)
def post(
self,
url: _t.UriType,
data: _t.DataType = None,
json: _t.JsonType = None,
**kwargs: Unpack[_t.PostKwargs],
) -> Response:
r"""Sends a POST request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) json to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
return self.request("POST", url, data=data, json=json, **kwargs)
def put(
self, url: _t.UriType, data: _t.DataType = None, **kwargs: Unpack[_t.DataKwargs]
) -> Response:
r"""Sends a PUT request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
return self.request("PUT", url, data=data, **kwargs)
def patch(
self, url: _t.UriType, data: _t.DataType = None, **kwargs: Unpack[_t.DataKwargs]
) -> Response:
r"""Sends a PATCH request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
return self.request("PATCH", url, data=data, **kwargs)
def delete(self, url: _t.UriType, **kwargs: Unpack[_t.RequestKwargs]) -> Response:
r"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
return self.request("DELETE", url, **kwargs)
def send(self, request: PreparedRequest, **kwargs: Any) -> Response:
"""Send a given PreparedRequest.
:rtype: requests.Response
"""
# Set defaults that the hooks can utilize to ensure they always have
# the correct parameters to reproduce the previous request.
kwargs.setdefault("stream", self.stream)
kwargs.setdefault("verify", self.verify)
kwargs.setdefault("cert", self.cert)
if "proxies" not in kwargs:
kwargs["proxies"] = resolve_proxies(request, self.proxies, self.trust_env)
# It's possible that users might accidentally send a Request object.
# Guard against that specific failure case.
if isinstance(request, Request):
raise ValueError("You can only send PreparedRequests.")
assert _is_prepared(request)
# Set up variables needed for resolve_redirects and dispatching of hooks
allow_redirects = kwargs.pop("allow_redirects", True)
stream = kwargs.get("stream")
hooks = request.hooks
# Get the appropriate adapter to use
adapter = self.get_adapter(url=request.url)
# Start time (approximately) of the request
start = preferred_clock()
# Send the request
r = adapter.send(request, **kwargs)
# Total elapsed time of the request (approximately)
elapsed = preferred_clock() - start
r.elapsed = timedelta(seconds=elapsed)
# Response manipulation hooks
r = dispatch_hook("response", hooks, r, **kwargs)
# Persist cookies
if r.history:
# If the hooks create history then we want those cookies too
for resp in r.history:
extract_cookies_to_jar(self.cookies, resp.request, resp.raw)
extract_cookies_to_jar(self.cookies, request, r.raw)
# Resolve redirects if allowed.
if allow_redirects:
# Redirect resolving generator.
gen = self.resolve_redirects(r, request, **kwargs)
history = [resp for resp in gen]
else:
history = []
# Shuffle things around if there's history.
if history:
# Insert the first (original) request at the start
history.insert(0, r)
# Get the last request made
r = history.pop()
r.history = history
# If redirects aren't being followed, store the response on the Request for Response.next().
if not allow_redirects:
try:
r._next = next( # type: ignore[assignment] # yield_requests=True returns PreparedRequest
self.resolve_redirects(r, request, yield_requests=True, **kwargs)
)
except StopIteration:
pass
if not stream:
r.content
return r
def merge_environment_settings(
self,
url: str,
proxies: dict[str, str] | None,
stream: bool | None,
verify: _t.VerifyType | None,
cert: _t.CertType,
) -> dict[str, Any]:
"""
Check the environment and merge it with some settings.
:rtype: dict
"""
# Gather clues from the surrounding environment.
if self.trust_env:
# Set environment's proxies.
no_proxy = proxies.get("no_proxy") if proxies is not None else None
env_proxies = get_environ_proxies(url, no_proxy=no_proxy)
if proxies is not None:
for k, v in env_proxies.items():
proxies.setdefault(k, v)
# Look for requests environment configuration
# and be compatible with cURL.
if verify is True or verify is None:
verify = (
os.environ.get("REQUESTS_CA_BUNDLE")
or os.environ.get("CURL_CA_BUNDLE")
or verify
)
# Merge all the kwargs.
proxies = merge_setting(proxies, self.proxies)
stream = merge_setting(stream, self.stream)
verify = merge_setting(verify, self.verify)
cert = merge_setting(cert, self.cert)
return {"proxies": proxies, "stream": stream, "verify": verify, "cert": cert}
def get_adapter(self, url: str) -> BaseAdapter:
"""
Returns the appropriate connection adapter for the given URL.
:rtype: requests.adapters.BaseAdapter
"""
for prefix, adapter in self.adapters.items():
if url.lower().startswith(prefix.lower()):
return adapter
# Nothing matches :-/
raise InvalidSchema(f"No connection adapters were found for {url!r}")
def close(self) -> None:
"""Closes all adapters and as such the session"""
for v in self.adapters.values():
v.close()
def mount(self, prefix: str, adapter: BaseAdapter) -> None:
"""Registers a connection adapter to a prefix.
Adapters are sorted in descending order by prefix length.
"""
self.adapters[prefix] = adapter
keys_to_move = [k for k in self.adapters if len(k) < len(prefix)]
for key in keys_to_move:
self.adapters[key] = self.adapters.pop(key)
def __getstate__(self) -> dict[str, Any]:
state = {attr: getattr(self, attr, None) for attr in self.__attrs__}
return state
def __setstate__(self, state: dict[str, Any]) -> None:
for attr, value in state.items():
setattr(self, attr, value)
def session() -> Session:
"""
Returns a :class:`Session` for context-management.
.. deprecated:: 1.0.0
This method has been deprecated since version 1.0.0 and is only kept for
backwards compatibility. New code should use :class:`~requests.sessions.Session`
to create a session. This may be removed at a future date.
:rtype: Session
"""
return Session()
@@ -0,0 +1,128 @@
r"""
The ``codes`` object defines a mapping from common names for HTTP statuses
to their numerical codes, accessible either as attributes or as dictionary
items.
Example::
>>> import requests
>>> requests.codes['temporary_redirect']
307
>>> requests.codes.teapot
418
>>> requests.codes['\o/']
200
Some codes have multiple names, and both upper- and lower-case versions of
the names are allowed. For example, ``codes.ok``, ``codes.OK``, and
``codes.okay`` all correspond to the HTTP status code 200.
"""
from .structures import LookupDict
_codes = {
# Informational.
100: ("continue",),
101: ("switching_protocols",),
102: ("processing", "early-hints"),
103: ("checkpoint",),
122: ("uri_too_long", "request_uri_too_long"),
200: ("ok", "okay", "all_ok", "all_okay", "all_good", "\\o/", "✓"),
201: ("created",),
202: ("accepted",),
203: ("non_authoritative_info", "non_authoritative_information"),
204: ("no_content",),
205: ("reset_content", "reset"),
206: ("partial_content", "partial"),
207: ("multi_status", "multiple_status", "multi_stati", "multiple_stati"),
208: ("already_reported",),
226: ("im_used",),
# Redirection.
300: ("multiple_choices",),
301: ("moved_permanently", "moved", "\\o-"),
302: ("found",),
303: ("see_other", "other"),
304: ("not_modified",),
305: ("use_proxy",),
306: ("switch_proxy",),
307: ("temporary_redirect", "temporary_moved", "temporary"),
308: (
"permanent_redirect",
"resume_incomplete",
"resume",
), # "resume" and "resume_incomplete" to be removed in 3.0
# Client Error.
400: ("bad_request", "bad"),
401: ("unauthorized",),
402: ("payment_required", "payment"),
403: ("forbidden",),
404: ("not_found", "-o-"),
405: ("method_not_allowed", "not_allowed"),
406: ("not_acceptable",),
407: ("proxy_authentication_required", "proxy_auth", "proxy_authentication"),
408: ("request_timeout", "timeout"),
409: ("conflict",),
410: ("gone",),
411: ("length_required",),
412: ("precondition_failed", "precondition"),
413: ("request_entity_too_large", "content_too_large"),
414: ("request_uri_too_large", "uri_too_long"),
415: ("unsupported_media_type", "unsupported_media", "media_type"),
416: (
"requested_range_not_satisfiable",
"requested_range",
"range_not_satisfiable",
),
417: ("expectation_failed",),
418: ("im_a_teapot", "teapot", "i_am_a_teapot"),
421: ("misdirected_request",),
422: ("unprocessable_entity", "unprocessable", "unprocessable_content"),
423: ("locked",),
424: ("failed_dependency", "dependency"),
425: ("unordered_collection", "unordered", "too_early"),
426: ("upgrade_required", "upgrade"),
428: ("precondition_required", "precondition"),
429: ("too_many_requests", "too_many"),
431: ("header_fields_too_large", "fields_too_large"),
444: ("no_response", "none"),
449: ("retry_with", "retry"),
450: ("blocked_by_windows_parental_controls", "parental_controls"),
451: ("unavailable_for_legal_reasons", "legal_reasons"),
499: ("client_closed_request",),
# Server Error.
500: ("internal_server_error", "server_error", "/o\\", "✗"),
501: ("not_implemented",),
502: ("bad_gateway",),
503: ("service_unavailable", "unavailable"),
504: ("gateway_timeout",),
505: ("http_version_not_supported", "http_version"),
506: ("variant_also_negotiates",),
507: ("insufficient_storage",),
509: ("bandwidth_limit_exceeded", "bandwidth"),
510: ("not_extended",),
511: ("network_authentication_required", "network_auth", "network_authentication"),
}
codes: LookupDict[int] = LookupDict(name="status_codes")
def _init():
for code, titles in _codes.items():
for title in titles:
setattr(codes, title, code)
if not title.startswith(("\\", "/")):
setattr(codes, title.upper(), code)
def doc(code: int) -> str:
names = ", ".join(f"``{n}``" for n in _codes[code])
return "* %d: %s" % (code, names)
global __doc__
__doc__ = (
__doc__ + "\n" + "\n".join(doc(code) for code in sorted(_codes))
if __doc__ is not None
else None
)
_init()
@@ -0,0 +1,130 @@
"""
requests.structures
~~~~~~~~~~~~~~~~~~~
Data structures that power Requests.
"""
from __future__ import annotations
from collections import OrderedDict
from collections.abc import Iterable, Iterator, Mapping
from typing import Any, Generic, TypeVar, overload
from .compat import MutableMapping
_VT = TypeVar("_VT")
_D = TypeVar("_D")
class CaseInsensitiveDict(MutableMapping[str, _VT], Generic[_VT]):
"""A case-insensitive ``dict``-like object.
Implements all methods and operations of
``MutableMapping`` as well as dict's ``copy``. Also
provides ``lower_items``.
All keys are expected to be strings. The structure remembers the
case of the last key to be set, and ``iter(instance)``,
``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
will contain case-sensitive keys. However, querying and contains
testing is case insensitive::
cid = CaseInsensitiveDict()
cid['Accept'] = 'application/json'
cid['aCCEPT'] == 'application/json' # True
list(cid) == ['Accept'] # True
For example, ``headers['content-encoding']`` will return the
value of a ``'Content-Encoding'`` response header, regardless
of how the header name was originally stored.
If the constructor, ``.update``, or equality comparison
operations are given keys that have equal ``.lower()``s, the
behavior is undefined.
"""
_store: OrderedDict[str, tuple[str, _VT]]
def __init__(
self,
data: Mapping[str, _VT] | Iterable[tuple[str, _VT]] | None = None,
**kwargs: _VT,
) -> None:
self._store = OrderedDict()
if data is None:
data = {}
self.update(data, **kwargs)
def __setitem__(self, key: str, value: _VT) -> None:
# Use the lowercased key for lookups, but store the actual
# key alongside the value.
self._store[key.lower()] = (key, value)
def __getitem__(self, key: str) -> _VT:
return self._store[key.lower()][1]
def __delitem__(self, key: str) -> None:
del self._store[key.lower()]
def __iter__(self) -> Iterator[str]:
return (casedkey for casedkey, _ in self._store.values())
def __len__(self) -> int:
return len(self._store)
def lower_items(self) -> Iterator[tuple[str, _VT]]:
"""Like iteritems(), but with all lowercase keys."""
return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items())
def __eq__(self, other: object) -> bool:
if isinstance(other, Mapping):
other_dict: CaseInsensitiveDict[Any] = CaseInsensitiveDict(other) # type: ignore[reportUnknownArgumentType]
else:
return NotImplemented
# Compare insensitively
return dict(self.lower_items()) == dict(other_dict.lower_items())
# Copy is required
def copy(self) -> CaseInsensitiveDict[_VT]:
return CaseInsensitiveDict(self._store.values())
def __repr__(self) -> str:
return str(dict(self.items()))
class LookupDict(dict[str, _VT]):
"""Dictionary lookup object."""
name: Any
def __init__(self, name: Any = None) -> None:
self.name = name
super().__init__()
def __repr__(self) -> str:
return f"<lookup '{self.name}'>"
def __getattr__(self, key: str) -> _VT | None:
# We need this for type checkers to infer typing
# on attribute access with status_codes.py
if key in self.__dict__:
return self.__dict__[key]
else:
raise AttributeError(
f"'{type(self).__name__}' object has no attribute '{key}'"
)
def __getitem__(self, key: str) -> _VT | None: # type: ignore[override]
# We allow fall-through here, so values default to None
return self.__dict__.get(key, None)
@overload
def get(self, key: str, default: None = None) -> _VT | None: ...
@overload
def get(self, key: str, default: _D | _VT) -> _D | _VT: ...
def get(self, key: str, default: _D | None = None) -> _VT | _D | None:
return self.__dict__.get(key, default)
@@ -0,0 +1,211 @@
"""
Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more
"""
from __future__ import annotations
# Set default logging handler to avoid "No handler found" warnings.
import logging
import sys
import typing
import warnings
from logging import NullHandler
from . import exceptions
from ._base_connection import _TYPE_BODY
from ._collections import HTTPHeaderDict
from ._version import __version__
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
from .response import BaseHTTPResponse, HTTPResponse
from .util.request import make_headers
from .util.retry import Retry
from .util.timeout import Timeout
# Ensure that Python is compiled with OpenSSL 1.1.1+
# If the 'ssl' module isn't available at all that's
# fine, we only care if the module is available.
try:
import ssl
except ImportError:
pass
else:
if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive:
warnings.warn(
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
"See: https://github.com/urllib3/urllib3/issues/3020",
exceptions.NotOpenSSLWarning,
)
elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive:
raise ImportError(
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
"See: https://github.com/urllib3/urllib3/issues/2168"
)
__author__ = "Andrey Petrov ([email protected])"
__license__ = "MIT"
__version__ = __version__
__all__ = (
"HTTPConnectionPool",
"HTTPHeaderDict",
"HTTPSConnectionPool",
"PoolManager",
"ProxyManager",
"HTTPResponse",
"Retry",
"Timeout",
"add_stderr_logger",
"connection_from_url",
"disable_warnings",
"encode_multipart_formdata",
"make_headers",
"proxy_from_url",
"request",
"BaseHTTPResponse",
)
logging.getLogger(__name__).addHandler(NullHandler())
def add_stderr_logger(
level: int = logging.DEBUG,
) -> logging.StreamHandler[typing.TextIO]:
"""
Helper for quickly adding a StreamHandler to the logger. Useful for
debugging.
Returns the handler after adding it.
"""
# This method needs to be in this __init__.py to get the __name__ correct
# even if urllib3 is vendored within another package.
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
logger.addHandler(handler)
logger.setLevel(level)
logger.debug("Added a stderr logging handler to logger: %s", __name__)
return handler
# ... Clean up.
del NullHandler
# All warning filters *must* be appended unless you're really certain that they
# shouldn't be: otherwise, it's very hard for users to use most Python
# mechanisms to silence them.
# SecurityWarning's always go off by default.
warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
def disable_warnings(category: type[Warning] = exceptions.HTTPWarning) -> None:
"""
Helper for quickly disabling all urllib3 warnings.
"""
warnings.simplefilter("ignore", category)
_DEFAULT_POOL = PoolManager()
def request(
method: str,
url: str,
*,
body: _TYPE_BODY | None = None,
fields: _TYPE_FIELDS | None = None,
headers: typing.Mapping[str, str] | None = None,
preload_content: bool | None = True,
decode_content: bool | None = True,
redirect: bool | None = True,
retries: Retry | bool | int | None = None,
timeout: Timeout | float | int | None = 3,
json: typing.Any | None = None,
) -> BaseHTTPResponse:
"""
A convenience, top-level request method. It uses a module-global ``PoolManager`` instance.
Therefore, its side effects could be shared across dependencies relying on it.
To avoid side effects create a new ``PoolManager`` instance and use it instead.
The method does not accept low-level ``**urlopen_kw`` keyword arguments.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param fields:
Data to encode and send in the request body.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param json:
Data to encode and send as JSON with UTF-encoded in the request body.
The ``"Content-Type"`` header will be set to ``"application/json"``
unless specified otherwise.
"""
return _DEFAULT_POOL.request(
method,
url,
body=body,
fields=fields,
headers=headers,
preload_content=preload_content,
decode_content=decode_content,
redirect=redirect,
retries=retries,
timeout=timeout,
json=json,
)
if sys.platform == "emscripten":
from .contrib.emscripten import inject_into_urllib3 # noqa: 401
inject_into_urllib3()
@@ -0,0 +1,167 @@
from __future__ import annotations
import typing
from .util.connection import _TYPE_SOCKET_OPTIONS
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
from .util.url import Url
_TYPE_BODY = typing.Union[
bytes, typing.IO[typing.Any], typing.Iterable[bytes | str], str
]
class ProxyConfig(typing.NamedTuple):
ssl_context: ssl.SSLContext | None
use_forwarding_for_https: bool
assert_hostname: None | str | typing.Literal[False]
assert_fingerprint: str | None
class _ResponseOptions(typing.NamedTuple):
# TODO: Remove this in favor of a better
# HTTP request/response lifecycle tracking.
request_method: str
request_url: str
preload_content: bool
decode_content: bool
enforce_content_length: bool
if typing.TYPE_CHECKING:
import ssl
from typing import Protocol
from .response import BaseHTTPResponse
class BaseHTTPConnection(Protocol):
default_port: typing.ClassVar[int]
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
host: str
port: int
timeout: None | (
float
) # Instance doesn't store _DEFAULT_TIMEOUT, must be resolved.
blocksize: int
source_address: tuple[str, int] | None
socket_options: _TYPE_SOCKET_OPTIONS | None
proxy: Url | None
proxy_config: ProxyConfig | None
is_verified: bool
proxy_is_verified: bool | None
def __init__(
self,
host: str,
port: int | None = None,
*,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
blocksize: int = 8192,
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
proxy: Url | None = None,
proxy_config: ProxyConfig | None = None,
) -> None: ...
def set_tunnel(
self,
host: str,
port: int | None = None,
headers: typing.Mapping[str, str] | None = None,
scheme: str = "http",
) -> None: ...
def connect(self) -> None: ...
def request(
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
# We know *at least* botocore is depending on the order of the
# first 3 parameters so to be safe we only mark the later ones
# as keyword-only to ensure we have space to extend.
*,
chunked: bool = False,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> None: ...
def getresponse(self) -> BaseHTTPResponse: ...
def close(self) -> None: ...
@property
def is_closed(self) -> bool:
"""Whether the connection either is brand new or has been previously closed.
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
properties must be False.
"""
@property
def is_connected(self) -> bool:
"""Whether the connection is actively connected to any origin (proxy or target)"""
@property
def has_connected_to_proxy(self) -> bool:
"""Whether the connection has successfully connected to its proxy.
This returns False if no proxy is in use. Used to determine whether
errors are coming from the proxy layer or from tunnelling to the target origin.
"""
class BaseHTTPSConnection(BaseHTTPConnection, Protocol):
default_port: typing.ClassVar[int]
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
# Certificate verification methods
cert_reqs: int | str | None
assert_hostname: None | str | typing.Literal[False]
assert_fingerprint: str | None
ssl_context: ssl.SSLContext | None
# Trusted CAs
ca_certs: str | None
ca_cert_dir: str | None
ca_cert_data: None | str | bytes
# TLS version
ssl_minimum_version: int | None
ssl_maximum_version: int | None
ssl_version: int | str | None # Deprecated
# Client certificates
cert_file: str | None
key_file: str | None
key_password: str | None
def __init__(
self,
host: str,
port: int | None = None,
*,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
blocksize: int = 16384,
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
proxy: Url | None = None,
proxy_config: ProxyConfig | None = None,
cert_reqs: int | str | None = None,
assert_hostname: None | str | typing.Literal[False] = None,
assert_fingerprint: str | None = None,
server_hostname: str | None = None,
ssl_context: ssl.SSLContext | None = None,
ca_certs: str | None = None,
ca_cert_dir: str | None = None,
ca_cert_data: None | str | bytes = None,
ssl_minimum_version: int | None = None,
ssl_maximum_version: int | None = None,
ssl_version: int | str | None = None, # Deprecated
cert_file: str | None = None,
key_file: str | None = None,
key_password: str | None = None,
) -> None: ...
@@ -0,0 +1,486 @@
from __future__ import annotations
import typing
from collections import OrderedDict
from enum import Enum, auto
from threading import RLock
if typing.TYPE_CHECKING:
# We can only import Protocol if TYPE_CHECKING because it's a development
# dependency, and is not available at runtime.
from typing import Protocol
from typing_extensions import Self
class HasGettableStringKeys(Protocol):
def keys(self) -> typing.Iterator[str]: ...
def __getitem__(self, key: str) -> str: ...
__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
# Key type
_KT = typing.TypeVar("_KT")
# Value type
_VT = typing.TypeVar("_VT")
# Default type
_DT = typing.TypeVar("_DT")
ValidHTTPHeaderSource = typing.Union[
"HTTPHeaderDict",
typing.Mapping[str, str],
typing.Iterable[tuple[str, str]],
"HasGettableStringKeys",
]
class _Sentinel(Enum):
not_passed = auto()
def ensure_can_construct_http_header_dict(
potential: object,
) -> ValidHTTPHeaderSource | None:
if isinstance(potential, HTTPHeaderDict):
return potential
elif isinstance(potential, typing.Mapping):
# Full runtime checking of the contents of a Mapping is expensive, so for the
# purposes of typechecking, we assume that any Mapping is the right shape.
return typing.cast(typing.Mapping[str, str], potential)
elif isinstance(potential, typing.Iterable):
# Similarly to Mapping, full runtime checking of the contents of an Iterable is
# expensive, so for the purposes of typechecking, we assume that any Iterable
# is the right shape.
return typing.cast(typing.Iterable[tuple[str, str]], potential)
elif hasattr(potential, "keys") and hasattr(potential, "__getitem__"):
return typing.cast("HasGettableStringKeys", potential)
else:
return None
class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT, _VT]):
"""
Provides a thread-safe dict-like container which maintains up to
``maxsize`` keys while throwing away the least-recently-used keys beyond
``maxsize``.
:param maxsize:
Maximum number of recent elements to retain.
:param dispose_func:
Every time an item is evicted from the container,
``dispose_func(value)`` is called. Callback which will get called
"""
_container: typing.OrderedDict[_KT, _VT]
_maxsize: int
dispose_func: typing.Callable[[_VT], None] | None
lock: RLock
def __init__(
self,
maxsize: int = 10,
dispose_func: typing.Callable[[_VT], None] | None = None,
) -> None:
super().__init__()
self._maxsize = maxsize
self.dispose_func = dispose_func
self._container = OrderedDict()
self.lock = RLock()
def __getitem__(self, key: _KT) -> _VT:
# Re-insert the item, moving it to the end of the eviction line.
with self.lock:
item = self._container.pop(key)
self._container[key] = item
return item
def __setitem__(self, key: _KT, value: _VT) -> None:
evicted_item = None
with self.lock:
# Possibly evict the existing value of 'key'
try:
# If the key exists, we'll overwrite it, which won't change the
# size of the pool. Because accessing a key should move it to
# the end of the eviction line, we pop it out first.
evicted_item = key, self._container.pop(key)
self._container[key] = value
except KeyError:
# When the key does not exist, we insert the value first so that
# evicting works in all cases, including when self._maxsize is 0
self._container[key] = value
if len(self._container) > self._maxsize:
# If we didn't evict an existing value, and we've hit our maximum
# size, then we have to evict the least recently used item from
# the beginning of the container.
evicted_item = self._container.popitem(last=False)
# After releasing the lock on the pool, dispose of any evicted value.
if evicted_item is not None and self.dispose_func:
_, evicted_value = evicted_item
self.dispose_func(evicted_value)
def __delitem__(self, key: _KT) -> None:
with self.lock:
value = self._container.pop(key)
if self.dispose_func:
self.dispose_func(value)
def __len__(self) -> int:
with self.lock:
return len(self._container)
def __iter__(self) -> typing.NoReturn:
raise NotImplementedError(
"Iteration over this class is unlikely to be threadsafe."
)
def clear(self) -> None:
with self.lock:
# Copy pointers to all values, then wipe the mapping
values = list(self._container.values())
self._container.clear()
if self.dispose_func:
for value in values:
self.dispose_func(value)
def keys(self) -> set[_KT]: # type: ignore[override]
with self.lock:
return set(self._container.keys())
class HTTPHeaderDictItemView(set[tuple[str, str]]):
"""
HTTPHeaderDict is unusual for a Mapping[str, str] in that it has two modes of
address.
If we directly try to get an item with a particular name, we will get a string
back that is the concatenated version of all the values:
>>> d['X-Header-Name']
'Value1, Value2, Value3'
However, if we iterate over an HTTPHeaderDict's items, we will optionally combine
these values based on whether combine=True was called when building up the dictionary
>>> d = HTTPHeaderDict({"A": "1", "B": "foo"})
>>> d.add("A", "2", combine=True)
>>> d.add("B", "bar")
>>> list(d.items())
[
('A', '1, 2'),
('B', 'foo'),
('B', 'bar'),
]
This class conforms to the interface required by the MutableMapping ABC while
also giving us the nonstandard iteration behavior we want; items with duplicate
keys, ordered by time of first insertion.
"""
_headers: HTTPHeaderDict
def __init__(self, headers: HTTPHeaderDict) -> None:
self._headers = headers
def __len__(self) -> int:
return len(list(self._headers.iteritems()))
def __iter__(self) -> typing.Iterator[tuple[str, str]]:
return self._headers.iteritems()
def __contains__(self, item: object) -> bool:
if isinstance(item, tuple) and len(item) == 2:
passed_key, passed_val = item
if isinstance(passed_key, str) and isinstance(passed_val, str):
return self._headers._has_value_for_header(passed_key, passed_val)
return False
class HTTPHeaderDict(typing.MutableMapping[str, str]):
"""
:param headers:
An iterable of field-value pairs. Must not contain multiple field names
when compared case-insensitively.
:param kwargs:
Additional field-value pairs to pass in to ``dict.update``.
A ``dict`` like container for storing HTTP Headers.
Field names are stored and compared case-insensitively in compliance with
RFC 7230. Iteration provides the first case-sensitive key seen for each
case-insensitive pair.
Using ``__setitem__`` syntax overwrites fields that compare equal
case-insensitively in order to maintain ``dict``'s api. For fields that
compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
in a loop.
If multiple fields that are equal case-insensitively are passed to the
constructor or ``.update``, the behavior is undefined and some will be
lost.
>>> headers = HTTPHeaderDict()
>>> headers.add('Set-Cookie', 'foo=bar')
>>> headers.add('set-cookie', 'baz=quxx')
>>> headers['content-length'] = '7'
>>> headers['SET-cookie']
'foo=bar, baz=quxx'
>>> headers['Content-Length']
'7'
"""
_container: typing.MutableMapping[str, list[str]]
def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str):
super().__init__()
self._container = {} # 'dict' is insert-ordered
if headers is not None:
if isinstance(headers, HTTPHeaderDict):
self._copy_from(headers)
else:
self.extend(headers)
if kwargs:
self.extend(kwargs)
def __setitem__(self, key: str, val: str) -> None:
# avoid a bytes/str comparison by decoding before httplib
if isinstance(key, bytes):
key = key.decode("latin-1")
self._container[key.lower()] = [key, val]
def __getitem__(self, key: str) -> str:
if isinstance(key, bytes):
key = key.decode("latin-1")
val = self._container[key.lower()]
return ", ".join(val[1:])
def __delitem__(self, key: str) -> None:
if isinstance(key, bytes):
key = key.decode("latin-1")
del self._container[key.lower()]
def __contains__(self, key: object) -> bool:
if isinstance(key, bytes):
key = key.decode("latin-1")
if isinstance(key, str):
return key.lower() in self._container
return False
def setdefault(self, key: str, default: str = "") -> str:
return super().setdefault(key, default)
def __eq__(self, other: object) -> bool:
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
return False
else:
other_as_http_header_dict = type(self)(maybe_constructable)
return {k.lower(): v for k, v in self.itermerged()} == {
k.lower(): v for k, v in other_as_http_header_dict.itermerged()
}
def __ne__(self, other: object) -> bool:
return not self.__eq__(other)
def __len__(self) -> int:
return len(self._container)
def __iter__(self) -> typing.Iterator[str]:
# Only provide the originally cased names
for vals in self._container.values():
yield vals[0]
def discard(self, key: str) -> None:
try:
del self[key]
except KeyError:
pass
def add(self, key: str, val: str, *, combine: bool = False) -> None:
"""Adds a (name, value) pair, doesn't overwrite the value if it already
exists.
If this is called with combine=True, instead of adding a new header value
as a distinct item during iteration, this will instead append the value to
any existing header value with a comma. If no existing header value exists
for the key, then the value will simply be added, ignoring the combine parameter.
>>> headers = HTTPHeaderDict(foo='bar')
>>> headers.add('Foo', 'baz')
>>> headers['foo']
'bar, baz'
>>> list(headers.items())
[('foo', 'bar'), ('foo', 'baz')]
>>> headers.add('foo', 'quz', combine=True)
>>> list(headers.items())
[('foo', 'bar, baz, quz')]
"""
# avoid a bytes/str comparison by decoding before httplib
if isinstance(key, bytes):
key = key.decode("latin-1")
key_lower = key.lower()
new_vals = [key, val]
# Keep the common case aka no item present as fast as possible
vals = self._container.setdefault(key_lower, new_vals)
if new_vals is not vals:
# if there are values here, then there is at least the initial
# key/value pair
assert len(vals) >= 2
if combine:
vals[-1] = vals[-1] + ", " + val
else:
vals.append(val)
def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None:
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError(
f"extend() takes at most 1 positional arguments ({len(args)} given)"
)
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, typing.Mapping):
for key, val in other.items():
self.add(key, val)
elif isinstance(other, typing.Iterable):
for key, value in other:
self.add(key, value)
elif hasattr(other, "keys") and hasattr(other, "__getitem__"):
# THIS IS NOT A TYPESAFE BRANCH
# In this branch, the object has a `keys` attr but is not a Mapping or any of
# the other types indicated in the method signature. We do some stuff with
# it as though it partially implements the Mapping interface, but we're not
# doing that stuff safely AT ALL.
for key in other.keys():
self.add(key, other[key])
for key, value in kwargs.items():
self.add(key, value)
@typing.overload
def getlist(self, key: str) -> list[str]: ...
@typing.overload
def getlist(self, key: str, default: _DT) -> list[str] | _DT: ...
def getlist(
self, key: str, default: _Sentinel | _DT = _Sentinel.not_passed
) -> list[str] | _DT:
"""Returns a list of all the values for the named field. Returns an
empty list if the key doesn't exist."""
if isinstance(key, bytes):
key = key.decode("latin-1")
try:
vals = self._container[key.lower()]
except KeyError:
if default is _Sentinel.not_passed:
# _DT is unbound; empty list is instance of List[str]
return []
# _DT is bound; default is instance of _DT
return default
else:
# _DT may or may not be bound; vals[1:] is instance of List[str], which
# meets our external interface requirement of `Union[List[str], _DT]`.
return vals[1:]
def _prepare_for_method_change(self) -> Self:
"""
Remove content-specific header fields before changing the request
method to GET or HEAD according to RFC 9110, Section 15.4.
"""
content_specific_headers = [
"Content-Encoding",
"Content-Language",
"Content-Location",
"Content-Type",
"Content-Length",
"Digest",
"Last-Modified",
]
for header in content_specific_headers:
self.discard(header)
return self
# Backwards compatibility for httplib
getheaders = getlist
getallmatchingheaders = getlist
iget = getlist
# Backwards compatibility for http.cookiejar
get_all = getlist
def __repr__(self) -> str:
return f"{type(self).__name__}({dict(self.itermerged())})"
def _copy_from(self, other: HTTPHeaderDict) -> None:
for key in other:
val = other.getlist(key)
self._container[key.lower()] = [key, *val]
def copy(self) -> Self:
clone = type(self)()
clone._copy_from(self)
return clone
def iteritems(self) -> typing.Iterator[tuple[str, str]]:
"""Iterate over all header lines, including duplicate ones."""
for key in self:
vals = self._container[key.lower()]
for val in vals[1:]:
yield vals[0], val
def itermerged(self) -> typing.Iterator[tuple[str, str]]:
"""Iterate over all headers, merging duplicate ones together."""
for key in self:
val = self._container[key.lower()]
yield val[0], ", ".join(val[1:])
def items(self) -> HTTPHeaderDictItemView: # type: ignore[override]
return HTTPHeaderDictItemView(self)
def _has_value_for_header(self, header_name: str, potential_value: str) -> bool:
if header_name in self:
return potential_value in self._container[header_name.lower()][1:]
return False
def __ior__(self, other: object) -> HTTPHeaderDict:
# Supports extending a header dict in-place using operator |=
# combining items with add instead of __setitem__
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
return NotImplemented
self.extend(maybe_constructable)
return self
def __or__(self, other: object) -> Self:
# Supports merging header dicts using operator |
# combining items with add instead of __setitem__
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
return NotImplemented
result = self.copy()
result.extend(maybe_constructable)
return result
def __ror__(self, other: object) -> Self:
# Supports merging header dicts using operator | when other is on left side
# combining items with add instead of __setitem__
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
return NotImplemented
result = type(self)(maybe_constructable)
result.extend(self)
return result
@@ -0,0 +1,278 @@
from __future__ import annotations
import json as _json
import typing
from urllib.parse import urlencode
from ._base_connection import _TYPE_BODY
from ._collections import HTTPHeaderDict
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
from .response import BaseHTTPResponse
__all__ = ["RequestMethods"]
_TYPE_ENCODE_URL_FIELDS = typing.Union[
typing.Sequence[tuple[str, typing.Union[str, bytes]]],
typing.Mapping[str, typing.Union[str, bytes]],
]
class RequestMethods:
"""
Convenience mixin for classes who implement a :meth:`urlopen` method, such
as :class:`urllib3.HTTPConnectionPool` and
:class:`urllib3.PoolManager`.
Provides behavior for making common types of HTTP request methods and
decides which type of request field encoding to use.
Specifically,
:meth:`.request_encode_url` is for sending requests whose fields are
encoded in the URL (such as GET, HEAD, DELETE).
:meth:`.request_encode_body` is for sending requests whose fields are
encoded in the *body* of the request using multipart or www-form-urlencoded
(such as for POST, PUT, PATCH).
:meth:`.request` is for making any kind of request, it will look up the
appropriate encoding format and use one of the above two methods to make
the request.
Initializer parameters:
:param headers:
Headers to include with all requests, unless other headers are given
explicitly.
"""
_encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None:
self.headers = headers or {}
def urlopen(
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
encode_multipart: bool = True,
multipart_boundary: str | None = None,
**kw: typing.Any,
) -> BaseHTTPResponse: # Abstract
raise NotImplementedError(
"Classes extending RequestMethods must implement "
"their own ``urlopen`` method."
)
def request(
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
fields: _TYPE_FIELDS | None = None,
headers: typing.Mapping[str, str] | None = None,
json: typing.Any | None = None,
**urlopen_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Make a request using :meth:`urlopen` with the appropriate encoding of
``fields`` based on the ``method`` used.
This is a convenience method that requires the least amount of manual
effort. It can be used in most situations, while still having the
option to drop down to more specific methods when necessary, such as
:meth:`request_encode_url`, :meth:`request_encode_body`,
or even the lowest level :meth:`urlopen`.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param fields:
Data to encode and send in the URL or request body, depending on ``method``.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param json:
Data to encode and send as JSON with UTF-encoded in the request body.
The ``"Content-Type"`` header will be set to ``"application/json"``
unless specified otherwise.
"""
method = method.upper()
if json is not None and body is not None:
raise TypeError(
"request got values for both 'body' and 'json' parameters which are mutually exclusive"
)
if json is not None:
if headers is None:
headers = self.headers
if not ("content-type" in map(str.lower, headers.keys())):
headers = HTTPHeaderDict(headers)
headers["Content-Type"] = "application/json"
body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode(
"utf-8"
)
if body is not None:
urlopen_kw["body"] = body
if method in self._encode_url_methods:
return self.request_encode_url(
method,
url,
fields=fields, # type: ignore[arg-type]
headers=headers,
**urlopen_kw,
)
else:
return self.request_encode_body(
method, url, fields=fields, headers=headers, **urlopen_kw
)
def request_encode_url(
self,
method: str,
url: str,
fields: _TYPE_ENCODE_URL_FIELDS | None = None,
headers: typing.Mapping[str, str] | None = None,
**urlopen_kw: str,
) -> BaseHTTPResponse:
"""
Make a request using :meth:`urlopen` with the ``fields`` encoded in
the url. This is useful for request methods like GET, HEAD, DELETE, etc.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param fields:
Data to encode and send in the URL.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
"""
if headers is None:
headers = self.headers
extra_kw: dict[str, typing.Any] = {"headers": headers}
extra_kw.update(urlopen_kw)
if fields:
url += "?" + urlencode(fields)
return self.urlopen(method, url, **extra_kw)
def request_encode_body(
self,
method: str,
url: str,
fields: _TYPE_FIELDS | None = None,
headers: typing.Mapping[str, str] | None = None,
encode_multipart: bool = True,
multipart_boundary: str | None = None,
**urlopen_kw: str,
) -> BaseHTTPResponse:
"""
Make a request using :meth:`urlopen` with the ``fields`` encoded in
the body. This is useful for request methods like POST, PUT, PATCH, etc.
When ``encode_multipart=True`` (default), then
:func:`urllib3.encode_multipart_formdata` is used to encode
the payload with the appropriate content type. Otherwise
:func:`urllib.parse.urlencode` is used with the
'application/x-www-form-urlencoded' content type.
Multipart encoding must be used when posting files, and it's reasonably
safe to use it in other times too. However, it may break request
signing, such as with OAuth.
Supports an optional ``fields`` parameter of key/value strings AND
key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
the MIME type is optional. For example::
fields = {
'foo': 'bar',
'fakefile': ('foofile.txt', 'contents of foofile'),
'realfile': ('barfile.txt', open('realfile').read()),
'typedfile': ('bazfile.bin', open('bazfile').read(),
'image/jpeg'),
'nonamefile': 'contents of nonamefile field',
}
When uploading a file, providing a filename (the first parameter of the
tuple) is optional but recommended to best mimic behavior of browsers.
Note that if ``headers`` are supplied, the 'Content-Type' header will
be overwritten because it depends on the dynamic random boundary string
which is used to compose the body of the request. The random boundary
string can be explicitly set with the ``multipart_boundary`` parameter.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param fields:
Data to encode and send in the request body.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param encode_multipart:
If True, encode the ``fields`` using the multipart/form-data MIME
format.
:param multipart_boundary:
If not specified, then a random boundary will be generated using
:func:`urllib3.filepost.choose_boundary`.
"""
if headers is None:
headers = self.headers
extra_kw: dict[str, typing.Any] = {"headers": HTTPHeaderDict(headers)}
body: bytes | str
if fields:
if "body" in urlopen_kw:
raise TypeError(
"request got values for both 'fields' and 'body', can only specify one."
)
if encode_multipart:
body, content_type = encode_multipart_formdata(
fields, boundary=multipart_boundary
)
else:
body, content_type = (
urlencode(fields), # type: ignore[arg-type]
"application/x-www-form-urlencoded",
)
extra_kw["body"] = body
extra_kw["headers"].setdefault("Content-Type", content_type)
extra_kw.update(urlopen_kw)
return self.urlopen(method, url, **extra_kw)
@@ -0,0 +1,24 @@
# file generated by vcs-versioning
# don't change, don't track in version control
from __future__ import annotations
__all__ = [
"__version__",
"__version_tuple__",
"version",
"version_tuple",
"__commit_id__",
"commit_id",
]
version: str
__version__: str
__version_tuple__: tuple[int | str, ...]
version_tuple: tuple[int | str, ...]
commit_id: str | None
__commit_id__: str | None
__version__ = version = '2.7.0'
__version_tuple__ = version_tuple = (2, 7, 0)
__commit_id__ = commit_id = None
@@ -0,0 +1,17 @@
from __future__ import annotations
import urllib3.connection
from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection
def inject_into_urllib3() -> None:
# override connection classes to use emscripten specific classes
# n.b. mypy complains about the overriding of classes below
# if it isn't ignored
HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection
HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection
urllib3.connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment]
urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment]
urllib3.connection.VerifiedHTTPSConnection = EmscriptenHTTPSConnection # type: ignore[assignment]
@@ -0,0 +1,260 @@
from __future__ import annotations
import os
import typing
# use http.client.HTTPException for consistency with non-emscripten
from http.client import HTTPException as HTTPException # noqa: F401
from http.client import ResponseNotReady
from ..._base_connection import _TYPE_BODY
from ...connection import HTTPConnection, ProxyConfig, port_by_scheme
from ...exceptions import TimeoutError
from ...response import BaseHTTPResponse
from ...util.connection import _TYPE_SOCKET_OPTIONS
from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
from ...util.url import Url
from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request
from .request import EmscriptenRequest
from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse
if typing.TYPE_CHECKING:
from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
class EmscriptenHTTPConnection:
default_port: typing.ClassVar[int] = port_by_scheme["http"]
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
timeout: None | (float)
host: str
port: int
blocksize: int
source_address: tuple[str, int] | None
socket_options: _TYPE_SOCKET_OPTIONS | None
proxy: Url | None
proxy_config: ProxyConfig | None
is_verified: bool = False
proxy_is_verified: bool | None = None
response_class: type[BaseHTTPResponse] = EmscriptenHttpResponseWrapper
_response: EmscriptenResponse | None
def __init__(
self,
host: str,
port: int = 0,
*,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
blocksize: int = 8192,
socket_options: _TYPE_SOCKET_OPTIONS | None = None,
proxy: Url | None = None,
proxy_config: ProxyConfig | None = None,
) -> None:
self.host = host
self.port = port
self.timeout = timeout if isinstance(timeout, float) else 0.0
self.scheme = "http"
self._closed = True
self._response = None
# ignore these things because we don't
# have control over that stuff
self.proxy = None
self.proxy_config = None
self.blocksize = blocksize
self.source_address = None
self.socket_options = None
self.is_verified = False
def set_tunnel(
self,
host: str,
port: int | None = 0,
headers: typing.Mapping[str, str] | None = None,
scheme: str = "http",
) -> None:
pass
def connect(self) -> None:
pass
def request(
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
# We know *at least* botocore is depending on the order of the
# first 3 parameters so to be safe we only mark the later ones
# as keyword-only to ensure we have space to extend.
*,
chunked: bool = False,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> None:
self._closed = False
if url.startswith("/"):
if self.port is not None:
port = f":{self.port}"
else:
port = ""
# no scheme / host / port included, make a full url
url = f"{self.scheme}://{self.host}{port}{url}"
request = EmscriptenRequest(
url=url,
method=method,
timeout=self.timeout if self.timeout else 0,
decode_content=decode_content,
)
request.set_body(body)
if headers:
for k, v in headers.items():
request.set_header(k, v)
self._response = None
try:
if not preload_content:
self._response = send_streaming_request(request)
if self._response is None:
self._response = send_request(request)
except _TimeoutError as e:
raise TimeoutError(e.message) from e
except _RequestError as e:
raise HTTPException(e.message) from e
def getresponse(self) -> BaseHTTPResponse:
if self._response is not None:
return EmscriptenHttpResponseWrapper(
internal_response=self._response,
url=self._response.request.url,
connection=self,
)
else:
raise ResponseNotReady()
def close(self) -> None:
self._closed = True
self._response = None
@property
def is_closed(self) -> bool:
"""Whether the connection either is brand new or has been previously closed.
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
properties must be False.
"""
return self._closed
@property
def is_connected(self) -> bool:
"""Whether the connection is actively connected to any origin (proxy or target)"""
return True
@property
def has_connected_to_proxy(self) -> bool:
"""Whether the connection has successfully connected to its proxy.
This returns False if no proxy is in use. Used to determine whether
errors are coming from the proxy layer or from tunnelling to the target origin.
"""
return False
class EmscriptenHTTPSConnection(EmscriptenHTTPConnection):
default_port = port_by_scheme["https"]
# all this is basically ignored, as browser handles https
cert_reqs: int | str | None = None
ca_certs: str | None = None
ca_cert_dir: str | None = None
ca_cert_data: None | str | bytes = None
cert_file: str | None
key_file: str | None
key_password: str | None
ssl_context: typing.Any | None
ssl_version: int | str | None = None
ssl_minimum_version: int | None = None
ssl_maximum_version: int | None = None
assert_hostname: None | str | typing.Literal[False]
assert_fingerprint: str | None = None
def __init__(
self,
host: str,
port: int = 0,
*,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
blocksize: int = 16384,
socket_options: (
None | _TYPE_SOCKET_OPTIONS
) = HTTPConnection.default_socket_options,
proxy: Url | None = None,
proxy_config: ProxyConfig | None = None,
cert_reqs: int | str | None = None,
assert_hostname: None | str | typing.Literal[False] = None,
assert_fingerprint: str | None = None,
server_hostname: str | None = None,
ssl_context: typing.Any | None = None,
ca_certs: str | None = None,
ca_cert_dir: str | None = None,
ca_cert_data: None | str | bytes = None,
ssl_minimum_version: int | None = None,
ssl_maximum_version: int | None = None,
ssl_version: int | str | None = None, # Deprecated
cert_file: str | None = None,
key_file: str | None = None,
key_password: str | None = None,
) -> None:
super().__init__(
host,
port=port,
timeout=timeout,
source_address=source_address,
blocksize=blocksize,
socket_options=socket_options,
proxy=proxy,
proxy_config=proxy_config,
)
self.scheme = "https"
self.key_file = key_file
self.cert_file = cert_file
self.key_password = key_password
self.ssl_context = ssl_context
self.server_hostname = server_hostname
self.assert_hostname = assert_hostname
self.assert_fingerprint = assert_fingerprint
self.ssl_version = ssl_version
self.ssl_minimum_version = ssl_minimum_version
self.ssl_maximum_version = ssl_maximum_version
self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
self.ca_cert_data = ca_cert_data
self.cert_reqs = None
# The browser will automatically verify all requests.
# We have no control over that setting.
self.is_verified = True
def set_cert(
self,
key_file: str | None = None,
cert_file: str | None = None,
cert_reqs: int | str | None = None,
key_password: str | None = None,
ca_certs: str | None = None,
assert_hostname: None | str | typing.Literal[False] = None,
assert_fingerprint: str | None = None,
ca_cert_dir: str | None = None,
ca_cert_data: None | str | bytes = None,
) -> None:
pass
# verify that this class implements BaseHTTP(s) connection correctly
if typing.TYPE_CHECKING:
_supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0)
_supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0)
@@ -0,0 +1,726 @@
"""
Support for streaming http requests in emscripten.
A few caveats -
If your browser (or Node.js) has WebAssembly JavaScript Promise Integration enabled
https://github.com/WebAssembly/js-promise-integration/blob/main/proposals/js-promise-integration/Overview.md
*and* you launch pyodide using `pyodide.runPythonAsync`, this will fetch data using the
JavaScript asynchronous fetch api (wrapped via `pyodide.ffi.call_sync`). In this case
timeouts and streaming should just work.
Otherwise, it uses a combination of XMLHttpRequest and a web-worker for streaming.
This approach has several caveats:
Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed.
Streaming only works if you're running pyodide in a web worker.
Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch
operation, so it requires that you have crossOriginIsolation enabled, by serving over https
(or from localhost) with the two headers below set:
Cross-Origin-Opener-Policy: same-origin
Cross-Origin-Embedder-Policy: require-corp
You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in
JavaScript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole
request into a buffer and then returning it. it shows a warning in the JavaScript console in this case.
Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once
control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch.
NB: in this code, there are a lot of JavaScript objects. They are named js_*
to make it clear what type of object they are.
"""
from __future__ import annotations
import io
import json
from email.parser import Parser
from importlib.resources import files
from typing import TYPE_CHECKING, Any
import js # type: ignore[import-not-found]
from pyodide.ffi import ( # type: ignore[import-not-found]
JsArray,
JsException,
JsProxy,
to_js,
)
if TYPE_CHECKING:
from typing_extensions import Buffer
from .request import EmscriptenRequest
from .response import EmscriptenResponse
"""
There are some headers that trigger unintended CORS preflight requests.
See also https://github.com/koenvo/pyodide-http/issues/22
"""
HEADERS_TO_IGNORE = ("user-agent",)
SUCCESS_HEADER = -1
SUCCESS_EOF = -2
ERROR_TIMEOUT = -3
ERROR_EXCEPTION = -4
class _RequestError(Exception):
def __init__(
self,
message: str | None = None,
*,
request: EmscriptenRequest | None = None,
response: EmscriptenResponse | None = None,
):
self.request = request
self.response = response
self.message = message
super().__init__(self.message)
class _StreamingError(_RequestError):
pass
class _TimeoutError(_RequestError):
pass
def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy:
return to_js(dict_val, dict_converter=js.Object.fromEntries)
class _ReadStream(io.RawIOBase):
def __init__(
self,
int_buffer: JsArray,
byte_buffer: JsArray,
timeout: float,
worker: JsProxy,
connection_id: int,
request: EmscriptenRequest,
):
self.int_buffer = int_buffer
self.byte_buffer = byte_buffer
self.read_pos = 0
self.read_len = 0
self.connection_id = connection_id
self.worker = worker
self.timeout = int(1000 * timeout) if timeout > 0 else None
self.is_live = True
self._is_closed = False
self.request: EmscriptenRequest | None = request
def __del__(self) -> None:
self.close()
# this is compatible with _base_connection
def is_closed(self) -> bool:
return self._is_closed
# for compatibility with RawIOBase
@property
def closed(self) -> bool:
return self.is_closed()
def close(self) -> None:
if self.is_closed():
return
self.read_len = 0
self.read_pos = 0
self.int_buffer = None
self.byte_buffer = None
self._is_closed = True
self.request = None
if self.is_live:
self.worker.postMessage(_obj_from_dict({"close": self.connection_id}))
self.is_live = False
super().close()
def readable(self) -> bool:
return True
def writable(self) -> bool:
return False
def seekable(self) -> bool:
return False
def readinto(self, byte_obj: Buffer) -> int:
if not self.int_buffer:
raise _StreamingError(
"No buffer for stream in _ReadStream.readinto",
request=self.request,
response=None,
)
if self.read_len == 0:
# wait for the worker to send something
js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT)
self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id}))
if (
js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout)
== "timed-out"
):
raise _TimeoutError
data_len = self.int_buffer[0]
if data_len > 0:
self.read_len = data_len
self.read_pos = 0
elif data_len == ERROR_EXCEPTION:
string_len = self.int_buffer[1]
# decode the error string
js_decoder = js.TextDecoder.new()
json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len))
raise _StreamingError(
f"Exception thrown in fetch: {json_str}",
request=self.request,
response=None,
)
else:
# EOF, free the buffers and return zero
# and free the request
self.is_live = False
self.close()
return 0
# copy from int32array to python bytes
ret_length = min(self.read_len, len(memoryview(byte_obj)))
subarray = self.byte_buffer.subarray(
self.read_pos, self.read_pos + ret_length
).to_py()
memoryview(byte_obj)[0:ret_length] = subarray
self.read_len -= ret_length
self.read_pos += ret_length
return ret_length
class _StreamingFetcher:
def __init__(self) -> None:
# make web-worker and data buffer on startup
self.streaming_ready = False
streaming_worker_code = (
files(__package__)
.joinpath("emscripten_fetch_worker.js")
.read_text(encoding="utf-8")
)
js_data_blob = js.Blob.new(
to_js([streaming_worker_code], create_pyproxies=False),
_obj_from_dict({"type": "application/javascript"}),
)
def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None:
def onMsg(e: JsProxy) -> None:
self.streaming_ready = True
js_resolve_fn(e)
def onErr(e: JsProxy) -> None:
js_reject_fn(e) # Defensive: never happens in ci
self.js_worker.onmessage = onMsg
self.js_worker.onerror = onErr
js_data_url = js.URL.createObjectURL(js_data_blob)
self.js_worker = js.globalThis.Worker.new(js_data_url)
self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver)
def send(self, request: EmscriptenRequest) -> EmscriptenResponse:
headers = {
k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE
}
body = request.body
fetch_data = {"headers": headers, "body": to_js(body), "method": request.method}
# start the request off in the worker
timeout = int(1000 * request.timeout) if request.timeout > 0 else None
js_shared_buffer = js.SharedArrayBuffer.new(1048576)
js_int_buffer = js.Int32Array.new(js_shared_buffer)
js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8)
js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT)
js.Atomics.notify(js_int_buffer, 0)
js_absolute_url = js.URL.new(request.url, js.location).href
self.js_worker.postMessage(
_obj_from_dict(
{
"buffer": js_shared_buffer,
"url": js_absolute_url,
"fetchParams": fetch_data,
}
)
)
# wait for the worker to send something
js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout)
if js_int_buffer[0] == ERROR_TIMEOUT:
raise _TimeoutError(
"Timeout connecting to streaming request",
request=request,
response=None,
)
elif js_int_buffer[0] == SUCCESS_HEADER:
# got response
# header length is in second int of intBuffer
string_len = js_int_buffer[1]
# decode the rest to a JSON string
js_decoder = js.TextDecoder.new()
# this does a copy (the slice) because decode can't work on shared array
# for some silly reason
json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
# get it as an object
response_obj = json.loads(json_str)
return EmscriptenResponse(
request=request,
status_code=response_obj["status"],
headers=response_obj["headers"],
body=_ReadStream(
js_int_buffer,
js_byte_buffer,
request.timeout,
self.js_worker,
response_obj["connectionID"],
request,
),
)
elif js_int_buffer[0] == ERROR_EXCEPTION:
string_len = js_int_buffer[1]
# decode the error string
js_decoder = js.TextDecoder.new()
json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
raise _StreamingError(
f"Exception thrown in fetch: {json_str}", request=request, response=None
)
else:
raise _StreamingError(
f"Unknown status from worker in fetch: {js_int_buffer[0]}",
request=request,
response=None,
)
class _JSPIReadStream(io.RawIOBase):
"""
A read stream that uses pyodide.ffi.run_sync to read from a JavaScript fetch
response. This requires support for WebAssembly JavaScript Promise Integration
in the containing browser, and for pyodide to be launched via runPythonAsync.
:param js_read_stream:
The JavaScript stream reader
:param timeout:
Timeout in seconds
:param request:
The request we're handling
:param response:
The response this stream relates to
:param js_abort_controller:
A JavaScript AbortController object, used for timeouts
"""
def __init__(
self,
js_read_stream: Any,
timeout: float,
request: EmscriptenRequest,
response: EmscriptenResponse,
js_abort_controller: Any, # JavaScript AbortController for timeouts
):
self.js_read_stream = js_read_stream
self.timeout = timeout
self._is_closed = False
self._is_done = False
self.request: EmscriptenRequest | None = request
self.response: EmscriptenResponse | None = response
self.current_buffer = None
self.current_buffer_pos = 0
self.js_abort_controller = js_abort_controller
def __del__(self) -> None:
self.close()
# this is compatible with _base_connection
def is_closed(self) -> bool:
return self._is_closed
# for compatibility with RawIOBase
@property
def closed(self) -> bool:
return self.is_closed()
def close(self) -> None:
if self.is_closed():
return
self.read_len = 0
self.read_pos = 0
self.js_read_stream.cancel()
self.js_read_stream = None
self._is_closed = True
self._is_done = True
self.request = None
self.response = None
super().close()
def readable(self) -> bool:
return True
def writable(self) -> bool:
return False
def seekable(self) -> bool:
return False
def _get_next_buffer(self) -> bool:
result_js = _run_sync_with_timeout(
self.js_read_stream.read(),
self.timeout,
self.js_abort_controller,
request=self.request,
response=self.response,
)
if result_js.done:
self._is_done = True
return False
else:
self.current_buffer = result_js.value.to_py()
self.current_buffer_pos = 0
return True
def readinto(self, byte_obj: Buffer) -> int:
if self.current_buffer is None:
if not self._get_next_buffer() or self.current_buffer is None:
self.close()
return 0
ret_length = min(
len(byte_obj), len(self.current_buffer) - self.current_buffer_pos
)
byte_obj[0:ret_length] = self.current_buffer[
self.current_buffer_pos : self.current_buffer_pos + ret_length
]
self.current_buffer_pos += ret_length
if self.current_buffer_pos == len(self.current_buffer):
self.current_buffer = None
return ret_length
# check if we are in a worker or not
def is_in_browser_main_thread() -> bool:
return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window
def is_cross_origin_isolated() -> bool:
return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated
def is_in_node() -> bool:
return (
hasattr(js, "process")
and hasattr(js.process, "release")
and hasattr(js.process.release, "name")
and js.process.release.name == "node"
)
def is_worker_available() -> bool:
return hasattr(js, "Worker") and hasattr(js, "Blob")
_fetcher: _StreamingFetcher | None = None
if is_worker_available() and (
(is_cross_origin_isolated() and not is_in_browser_main_thread())
and (not is_in_node())
):
_fetcher = _StreamingFetcher()
else:
_fetcher = None
NODE_JSPI_ERROR = (
"urllib3 only works in Node.js with pyodide.runPythonAsync"
" and requires the flag --experimental-wasm-stack-switching in "
" versions of node <24."
)
def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None:
if has_jspi():
return send_jspi_request(request, True)
elif is_in_node():
raise _RequestError(
message=NODE_JSPI_ERROR,
request=request,
response=None,
)
if _fetcher and streaming_ready():
return _fetcher.send(request)
else:
_show_streaming_warning()
return None
_SHOWN_TIMEOUT_WARNING = False
def _show_timeout_warning() -> None:
global _SHOWN_TIMEOUT_WARNING
if not _SHOWN_TIMEOUT_WARNING:
_SHOWN_TIMEOUT_WARNING = True
message = "Warning: Timeout is not available on main browser thread"
js.console.warn(message)
_SHOWN_STREAMING_WARNING = False
def _show_streaming_warning() -> None:
global _SHOWN_STREAMING_WARNING
if not _SHOWN_STREAMING_WARNING:
_SHOWN_STREAMING_WARNING = True
message = "Can't stream HTTP requests because: \n"
if not is_cross_origin_isolated():
message += " Page is not cross-origin isolated\n"
if is_in_browser_main_thread():
message += " Python is running in main browser thread\n"
if not is_worker_available():
message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in
if streaming_ready() is False:
message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch
is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`"""
from js import console
console.warn(message)
def send_request(request: EmscriptenRequest) -> EmscriptenResponse:
if has_jspi():
return send_jspi_request(request, False)
elif is_in_node():
raise _RequestError(
message=NODE_JSPI_ERROR,
request=request,
response=None,
)
try:
js_xhr = js.XMLHttpRequest.new()
if not is_in_browser_main_thread():
js_xhr.responseType = "arraybuffer"
if request.timeout:
js_xhr.timeout = int(request.timeout * 1000)
else:
js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15")
if request.timeout:
# timeout isn't available on the main thread - show a warning in console
# if it is set
_show_timeout_warning()
js_xhr.open(request.method, request.url, False)
for name, value in request.headers.items():
if name.lower() not in HEADERS_TO_IGNORE:
js_xhr.setRequestHeader(name, value)
js_xhr.send(to_js(request.body))
headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders()))
if not is_in_browser_main_thread():
body = js_xhr.response.to_py().tobytes()
else:
body = js_xhr.response.encode("ISO-8859-15")
return EmscriptenResponse(
status_code=js_xhr.status, headers=headers, body=body, request=request
)
except JsException as err:
if err.name == "TimeoutError":
raise _TimeoutError(err.message, request=request)
elif err.name == "NetworkError":
raise _RequestError(err.message, request=request)
else:
# general http error
raise _RequestError(err.message, request=request)
def send_jspi_request(
request: EmscriptenRequest, streaming: bool
) -> EmscriptenResponse:
"""
Send a request using WebAssembly JavaScript Promise Integration
to wrap the asynchronous JavaScript fetch api (experimental).
:param request:
Request to send
:param streaming:
Whether to stream the response
:return: The response object
:rtype: EmscriptenResponse
"""
timeout = request.timeout
js_abort_controller = js.AbortController.new()
headers = {k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE}
req_body = request.body
fetch_data = {
"headers": headers,
"body": to_js(req_body),
"method": request.method,
"signal": js_abort_controller.signal,
}
# Node.js returns the whole response (unlike opaqueredirect in browsers),
# so urllib3 can set `redirect: manual` to control redirects itself.
# https://stackoverflow.com/a/78524615
if _is_node_js():
fetch_data["redirect"] = "manual"
# Call JavaScript fetch (async api, returns a promise)
fetcher_promise_js = js.fetch(request.url, _obj_from_dict(fetch_data))
# Now suspend WebAssembly until we resolve that promise
# or time out.
response_js = _run_sync_with_timeout(
fetcher_promise_js,
timeout,
js_abort_controller,
request=request,
response=None,
)
headers = {}
header_iter = response_js.headers.entries()
while True:
iter_value_js = header_iter.next()
if getattr(iter_value_js, "done", False):
break
else:
headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1])
status_code = response_js.status
body: bytes | io.RawIOBase = b""
response = EmscriptenResponse(
status_code=status_code, headers=headers, body=b"", request=request
)
if streaming:
# get via inputstream
if response_js.body is not None:
# get a reader from the fetch response
body_stream_js = response_js.body.getReader()
body = _JSPIReadStream(
body_stream_js, timeout, request, response, js_abort_controller
)
else:
# get directly via arraybuffer
# n.b. this is another async JavaScript call.
body = _run_sync_with_timeout(
response_js.arrayBuffer(),
timeout,
js_abort_controller,
request=request,
response=response,
).to_py()
response.body = body
return response
def _run_sync_with_timeout(
promise: Any,
timeout: float,
js_abort_controller: Any,
request: EmscriptenRequest | None,
response: EmscriptenResponse | None,
) -> Any:
"""
Await a JavaScript promise synchronously with a timeout which is implemented
via the AbortController
:param promise:
Javascript promise to await
:param timeout:
Timeout in seconds
:param js_abort_controller:
A JavaScript AbortController object, used on timeout
:param request:
The request being handled
:param response:
The response being handled (if it exists yet)
:raises _TimeoutError: If the request times out
:raises _RequestError: If the request raises a JavaScript exception
:return: The result of awaiting the promise.
"""
timer_id = None
if timeout > 0:
timer_id = js.setTimeout(
js_abort_controller.abort.bind(js_abort_controller), int(timeout * 1000)
)
try:
from pyodide.ffi import run_sync
# run_sync here uses WebAssembly JavaScript Promise Integration to
# suspend python until the JavaScript promise resolves.
return run_sync(promise)
except JsException as err:
if err.name == "AbortError":
raise _TimeoutError(
message="Request timed out", request=request, response=response
)
else:
raise _RequestError(message=err.message, request=request, response=response)
finally:
if timer_id is not None:
js.clearTimeout(timer_id)
def has_jspi() -> bool:
"""
Return true if jspi can be used.
This requires both browser support and also WebAssembly
to be in the correct state - i.e. that the javascript
call into python was async not sync.
:return: True if jspi can be used.
:rtype: bool
"""
try:
from pyodide.ffi import can_run_sync, run_sync # noqa: F401
return bool(can_run_sync())
except ImportError:
return False
def _is_node_js() -> bool:
"""
Check if we are in Node.js.
:return: True if we are in Node.js.
:rtype: bool
"""
return (
hasattr(js, "process")
and hasattr(js.process, "release")
# According to the Node.js documentation, the release name is always "node".
and js.process.release.name == "node"
)
def streaming_ready() -> bool | None:
if _fetcher:
return _fetcher.streaming_ready
else:
return None # no fetcher, return None to signify that
async def wait_for_streaming_ready() -> bool:
if _fetcher:
await _fetcher.js_worker_ready_promise
return True
else:
return False
@@ -0,0 +1,22 @@
from __future__ import annotations
from dataclasses import dataclass, field
from ..._base_connection import _TYPE_BODY
@dataclass
class EmscriptenRequest:
method: str
url: str
params: dict[str, str] | None = None
body: _TYPE_BODY | None = None
headers: dict[str, str] = field(default_factory=dict)
timeout: float = 0
decode_content: bool = True
def set_header(self, name: str, value: str) -> None:
self.headers[name.capitalize()] = value
def set_body(self, body: _TYPE_BODY | None) -> None:
self.body = body
@@ -0,0 +1,281 @@
from __future__ import annotations
import json as _json
import logging
import typing
from contextlib import contextmanager
from dataclasses import dataclass
from http.client import HTTPException as HTTPException
from io import BytesIO, IOBase
from ...exceptions import InvalidHeader, TimeoutError
from ...response import BaseHTTPResponse
from ...util.retry import Retry
from .request import EmscriptenRequest
if typing.TYPE_CHECKING:
from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
log = logging.getLogger(__name__)
@dataclass
class EmscriptenResponse:
status_code: int
headers: dict[str, str]
body: IOBase | bytes
request: EmscriptenRequest
class EmscriptenHttpResponseWrapper(BaseHTTPResponse):
def __init__(
self,
internal_response: EmscriptenResponse,
url: str | None = None,
connection: BaseHTTPConnection | BaseHTTPSConnection | None = None,
):
self._pool = None # set by pool class
self._body = None
self._uncached_read_occurred = False
self._response = internal_response
self._url = url
self._connection = connection
self._closed = False
super().__init__(
headers=internal_response.headers,
status=internal_response.status_code,
request_url=url,
version=0,
version_string="HTTP/?",
reason="",
decode_content=True,
)
self.length_remaining = self._init_length(self._response.request.method)
self.length_is_certain = False
@property
def url(self) -> str | None:
return self._url
@url.setter
def url(self, url: str | None) -> None:
self._url = url
@property
def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None:
return self._connection
@property
def retries(self) -> Retry | None:
return self._retries
@retries.setter
def retries(self, retries: Retry | None) -> None:
# Override the request_url if retries has a redirect location.
self._retries = retries
def stream(
self, amt: int | None = 2**16, decode_content: bool | None = None
) -> typing.Generator[bytes]:
"""
A generator wrapper for the read() method. A call will block until
``amt`` bytes have been read from the connection or until the
connection is closed.
:param amt:
How much of the content to read. The generator will return up to
much data per iteration, but may return less. This is particularly
likely when using compressed data. However, the empty string will
never be returned.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
"""
while True:
data = self.read(amt=amt, decode_content=decode_content)
if data:
yield data
else:
break
def _init_length(self, request_method: str | None) -> int | None:
length: int | None
content_length: str | None = self.headers.get("content-length")
if content_length is not None:
try:
# RFC 7230 section 3.3.2 specifies multiple content lengths can
# be sent in a single Content-Length header
# (e.g. Content-Length: 42, 42). This line ensures the values
# are all valid ints and that as long as the `set` length is 1,
# all values are the same. Otherwise, the header is invalid.
lengths = {int(val) for val in content_length.split(",")}
if len(lengths) > 1:
raise InvalidHeader(
"Content-Length contained multiple "
"unmatching values (%s)" % content_length
)
length = lengths.pop()
except ValueError:
length = None
else:
if length < 0:
length = None
else: # if content_length is None
length = None
# Check for responses that shouldn't include a body
if (
self.status in (204, 304)
or 100 <= self.status < 200
or request_method == "HEAD"
):
length = 0
return length
def read(
self,
amt: int | None = None,
decode_content: bool | None = None, # ignored because browser decodes always
cache_content: bool = False,
) -> bytes:
if (
self._closed
or self._response is None
or (isinstance(self._response.body, IOBase) and self._response.body.closed)
):
return b""
with self._error_catcher():
# body has been preloaded as a string by XmlHttpRequest
if not isinstance(self._response.body, IOBase):
self.length_remaining = len(self._response.body)
self.length_is_certain = True
# wrap body in IOStream
self._response.body = BytesIO(self._response.body)
if amt is not None and amt >= 0:
# don't cache partial content
cache_content = False
data = self._response.body.read(amt)
self._uncached_read_occurred = True
else: # read all we can (and cache it)
data = self._response.body.read()
if cache_content and not self._uncached_read_occurred:
self._body = data
else:
self._uncached_read_occurred = True
if self.length_remaining is not None:
self.length_remaining = max(self.length_remaining - len(data), 0)
if len(data) == 0 or (
self.length_is_certain and self.length_remaining == 0
):
# definitely finished reading, close response stream
self._response.body.close()
return typing.cast(bytes, data)
def read_chunked(
self,
amt: int | None = None,
decode_content: bool | None = None,
) -> typing.Generator[bytes]:
# chunked is handled by browser
while True:
bytes = self.read(amt, decode_content)
if not bytes:
break
yield bytes
def release_conn(self) -> None:
if not self._pool or not self._connection:
return None
self._pool._put_conn(self._connection)
self._connection = None
def drain_conn(self) -> None:
self.close()
@property
def data(self) -> bytes:
if self._body:
return self._body
else:
return self.read(cache_content=True)
def json(self) -> typing.Any:
"""
Deserializes the body of the HTTP response as a Python object.
The body of the HTTP response must be encoded using UTF-8, as per
`RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
your custom decoder instead.
If the body of the HTTP response is not decodable to UTF-8, a
`UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
valid JSON document, a `json.JSONDecodeError` will be raised.
Read more :ref:`here <json_content>`.
:returns: The body of the HTTP response as a Python object.
"""
data = self.data.decode("utf-8")
return _json.loads(data)
def close(self) -> None:
if not self._closed:
if isinstance(self._response.body, IOBase):
self._response.body.close()
if self._connection:
self._connection.close()
self._connection = None
self._closed = True
@contextmanager
def _error_catcher(self) -> typing.Generator[None]:
"""
Catch Emscripten specific exceptions thrown by fetch.py,
instead re-raising urllib3 variants, so that low-level exceptions
are not leaked in the high-level api.
On exit, release the connection back to the pool.
"""
from .fetch import _RequestError, _TimeoutError # avoid circular import
clean_exit = False
try:
yield
# If no exception is thrown, we should avoid cleaning up
# unnecessarily.
clean_exit = True
except _TimeoutError as e:
raise TimeoutError(str(e))
except _RequestError as e:
raise HTTPException(str(e))
finally:
# If we didn't terminate cleanly, we need to throw away our
# connection.
if not clean_exit:
# The response may not be closed but we're not going to use it
# anymore so close it now
if (
isinstance(self._response.body, IOBase)
and not self._response.body.closed
):
self._response.body.close()
# release the connection back to the pool
self.release_conn()
else:
# If we have read everything from the response stream,
# return the connection back to the pool.
if (
isinstance(self._response.body, IOBase)
and self._response.body.closed
):
self.release_conn()
@@ -0,0 +1,563 @@
"""
Module for using pyOpenSSL as a TLS backend. This module was relevant before
the standard library ``ssl`` module supported SNI, but now that we've dropped
support for Python 2.7 all relevant Python versions support SNI so
**this module is no longer recommended**.
This needs the following packages installed:
* `pyOpenSSL`_ (tested with 19.0.0)
* `cryptography`_ (minimum 2.3, from pyopenssl)
* `idna`_ (minimum 2.1, from cryptography)
However, pyOpenSSL depends on cryptography, so while we use all three directly here we
end up having relatively few packages required.
You can install them with the following command:
.. code-block:: bash
$ python -m pip install pyopenssl cryptography idna
To activate certificate checking, call
:func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code
before you begin making HTTP requests. This can be done in a ``sitecustomize``
module, or at any other time before your application begins using ``urllib3``,
like this:
.. code-block:: python
try:
import urllib3.contrib.pyopenssl
urllib3.contrib.pyopenssl.inject_into_urllib3()
except ImportError:
pass
.. _pyopenssl: https://www.pyopenssl.org
.. _cryptography: https://cryptography.io
.. _idna: https://github.com/kjd/idna
"""
from __future__ import annotations
import OpenSSL.SSL # type: ignore[import-not-found]
from cryptography import x509
try:
from cryptography.x509 import UnsupportedExtension # type: ignore[attr-defined]
except ImportError:
# UnsupportedExtension is gone in cryptography >= 2.1.0
class UnsupportedExtension(Exception): # type: ignore[no-redef]
pass
import logging
import ssl
import typing
from io import BytesIO
from socket import socket as socket_cls
from .. import util
if typing.TYPE_CHECKING:
from OpenSSL.crypto import X509 # type: ignore[import-not-found]
__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
# Map from urllib3 to PyOpenSSL compatible parameter-values.
_openssl_versions: dict[int, int] = {
util.ssl_.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
util.ssl_.PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
}
if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
_openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
_openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
_stdlib_to_openssl_verify = {
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
+ OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
}
_openssl_to_stdlib_verify = {v: k for k, v in _stdlib_to_openssl_verify.items()}
# The SSLvX values are the most likely to be missing in the future
# but we check them all just to be sure.
_OP_NO_SSLv2_OR_SSLv3: int = getattr(OpenSSL.SSL, "OP_NO_SSLv2", 0) | getattr(
OpenSSL.SSL, "OP_NO_SSLv3", 0
)
_OP_NO_TLSv1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1", 0)
_OP_NO_TLSv1_1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_1", 0)
_OP_NO_TLSv1_2: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_2", 0)
_OP_NO_TLSv1_3: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_3", 0)
_openssl_to_ssl_minimum_version: dict[int, int] = {
ssl.TLSVersion.MINIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
ssl.TLSVersion.TLSv1: _OP_NO_SSLv2_OR_SSLv3,
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1,
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1,
ssl.TLSVersion.TLSv1_3: (
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
),
ssl.TLSVersion.MAXIMUM_SUPPORTED: (
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
),
}
_openssl_to_ssl_maximum_version: dict[int, int] = {
ssl.TLSVersion.MINIMUM_SUPPORTED: (
_OP_NO_SSLv2_OR_SSLv3
| _OP_NO_TLSv1
| _OP_NO_TLSv1_1
| _OP_NO_TLSv1_2
| _OP_NO_TLSv1_3
),
ssl.TLSVersion.TLSv1: (
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3
),
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3,
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_3,
ssl.TLSVersion.TLSv1_3: _OP_NO_SSLv2_OR_SSLv3,
ssl.TLSVersion.MAXIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
}
# OpenSSL will only write 16K at a time
SSL_WRITE_BLOCKSIZE = 16384
orig_util_SSLContext = util.ssl_.SSLContext
log = logging.getLogger(__name__)
def inject_into_urllib3() -> None:
"Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
_validate_dependencies_met()
util.SSLContext = PyOpenSSLContext # type: ignore[assignment]
util.ssl_.SSLContext = PyOpenSSLContext # type: ignore[assignment]
util.IS_PYOPENSSL = True
util.ssl_.IS_PYOPENSSL = True
def extract_from_urllib3() -> None:
"Undo monkey-patching by :func:`inject_into_urllib3`."
util.SSLContext = orig_util_SSLContext
util.ssl_.SSLContext = orig_util_SSLContext
util.IS_PYOPENSSL = False
util.ssl_.IS_PYOPENSSL = False
def _validate_dependencies_met() -> None:
"""
Verifies that PyOpenSSL's package-level dependencies have been met.
Throws `ImportError` if they are not met.
"""
# Method added in `cryptography==1.1`; not available in older versions
from cryptography.x509.extensions import Extensions
if getattr(Extensions, "get_extension_for_class", None) is None:
raise ImportError(
"'cryptography' module missing required functionality. "
"Try upgrading to v1.3.4 or newer."
)
# pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
# attribute is only present on those versions.
from OpenSSL.crypto import X509
x509 = X509()
if getattr(x509, "_x509", None) is None:
raise ImportError(
"'pyOpenSSL' module missing required functionality. "
"Try upgrading to v0.14 or newer."
)
def _dnsname_to_stdlib(name: str) -> str | None:
"""
Converts a dNSName SubjectAlternativeName field to the form used by the
standard library on the given Python version.
Cryptography produces a dNSName as a unicode string that was idna-decoded
from ASCII bytes. We need to idna-encode that string to get it back, and
then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib
uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8).
If the name cannot be idna-encoded then we return None signalling that
the name given should be skipped.
"""
def idna_encode(name: str) -> bytes | None:
"""
Borrowed wholesale from the Python Cryptography Project. It turns out
that we can't just safely call `idna.encode`: it can explode for
wildcard names. This avoids that problem.
"""
import idna
try:
for prefix in ["*.", "."]:
if name.startswith(prefix):
name = name[len(prefix) :]
return prefix.encode("ascii") + idna.encode(name)
return idna.encode(name)
except idna.core.IDNAError:
return None
# Don't send IPv6 addresses through the IDNA encoder.
if ":" in name:
return name
encoded_name = idna_encode(name)
if encoded_name is None:
return None
return encoded_name.decode("utf-8")
def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]:
"""
Given an PyOpenSSL certificate, provides all the subject alternative names.
"""
cert = peer_cert.to_cryptography()
# We want to find the SAN extension. Ask Cryptography to locate it (it's
# faster than looping in Python)
try:
ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
except x509.ExtensionNotFound:
# No such extension, return the empty list.
return []
except (
x509.DuplicateExtension,
UnsupportedExtension,
x509.UnsupportedGeneralNameType,
UnicodeError,
) as e:
# A problem has been found with the quality of the certificate. Assume
# no SAN field is present.
log.warning(
"A problem was encountered with the certificate that prevented "
"urllib3 from finding the SubjectAlternativeName field. This can "
"affect certificate validation. The error was %s",
e,
)
return []
# We want to return dNSName and iPAddress fields. We need to cast the IPs
# back to strings because the match_hostname function wants them as
# strings.
# Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8
# decoded. This is pretty frustrating, but that's what the standard library
# does with certificates, and so we need to attempt to do the same.
# We also want to skip over names which cannot be idna encoded.
names = [
("DNS", name)
for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName))
if name is not None
]
names.extend(
("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress)
)
return names
class WrappedSocket:
"""API-compatibility wrapper for Python OpenSSL's Connection-class."""
def __init__(
self,
connection: OpenSSL.SSL.Connection,
socket: socket_cls,
suppress_ragged_eofs: bool = True,
) -> None:
self.connection = connection
self.socket = socket
self.suppress_ragged_eofs = suppress_ragged_eofs
self._io_refs = 0
self._closed = False
def fileno(self) -> int:
return self.socket.fileno()
# Copy-pasted from Python 3.5 source code
def _decref_socketios(self) -> None:
if self._io_refs > 0:
self._io_refs -= 1
if self._closed:
self.close()
def recv(self, *args: typing.Any, **kwargs: typing.Any) -> bytes:
try:
data = self.connection.recv(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return b""
else:
raise OSError(e.args[0], str(e)) from e
except OpenSSL.SSL.ZeroReturnError:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b""
else:
raise
except OpenSSL.SSL.WantReadError as e:
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise TimeoutError("The read operation timed out") from e
else:
return self.recv(*args, **kwargs)
# TLS 1.3 post-handshake authentication
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"read error: {e!r}") from e
else:
return data # type: ignore[no-any-return]
def recv_into(self, *args: typing.Any, **kwargs: typing.Any) -> int:
try:
return self.connection.recv_into(*args, **kwargs) # type: ignore[no-any-return]
except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return 0
else:
raise OSError(e.args[0], str(e)) from e
except OpenSSL.SSL.ZeroReturnError:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return 0
else:
raise
except OpenSSL.SSL.WantReadError as e:
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise TimeoutError("The read operation timed out") from e
else:
return self.recv_into(*args, **kwargs)
# TLS 1.3 post-handshake authentication
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"read error: {e!r}") from e
def settimeout(self, timeout: float) -> None:
return self.socket.settimeout(timeout)
def _send_until_done(self, data: bytes) -> int:
while True:
try:
return self.connection.send(data) # type: ignore[no-any-return]
except OpenSSL.SSL.WantWriteError as e:
if not util.wait_for_write(self.socket, self.socket.gettimeout()):
raise TimeoutError() from e
continue
except OpenSSL.SSL.SysCallError as e:
raise OSError(e.args[0], str(e)) from e
def sendall(self, data: bytes) -> None:
total_sent = 0
while total_sent < len(data):
sent = self._send_until_done(
data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]
)
total_sent += sent
def shutdown(self, how: int) -> None:
try:
self.connection.shutdown()
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"shutdown error: {e!r}") from e
def close(self) -> None:
self._closed = True
if self._io_refs <= 0:
self._real_close()
def _real_close(self) -> None:
try:
return self.connection.close() # type: ignore[no-any-return]
except OpenSSL.SSL.Error:
return
def getpeercert(
self, binary_form: bool = False
) -> dict[str, list[typing.Any]] | None:
x509 = self.connection.get_peer_certificate()
if not x509:
return x509 # type: ignore[no-any-return]
if binary_form:
return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) # type: ignore[no-any-return]
return {
"subject": ((("commonName", x509.get_subject().CN),),), # type: ignore[dict-item]
"subjectAltName": get_subj_alt_name(x509),
}
def version(self) -> str:
return self.connection.get_protocol_version_name() # type: ignore[no-any-return]
def selected_alpn_protocol(self) -> str | None:
alpn_proto = self.connection.get_alpn_proto_negotiated()
return alpn_proto.decode() if alpn_proto else None
WrappedSocket.makefile = socket_cls.makefile # type: ignore[attr-defined]
class PyOpenSSLContext:
"""
I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
for translating the interface of the standard library ``SSLContext`` object
to calls into PyOpenSSL.
"""
def __init__(self, protocol: int) -> None:
self.protocol = _openssl_versions[protocol]
self._ctx = OpenSSL.SSL.Context(self.protocol)
self._options = 0
self.check_hostname = False
self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED
self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED
self._verify_flags: int = ssl.VERIFY_X509_TRUSTED_FIRST
@property
def options(self) -> int:
return self._options
@options.setter
def options(self, value: int) -> None:
self._options = value
self._set_ctx_options()
@property
def verify_flags(self) -> int:
return self._verify_flags
@verify_flags.setter
def verify_flags(self, value: int) -> None:
self._verify_flags = value
self._ctx.get_cert_store().set_flags(self._verify_flags)
@property
def verify_mode(self) -> int:
return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
@verify_mode.setter
def verify_mode(self, value: ssl.VerifyMode) -> None:
self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
def set_default_verify_paths(self) -> None:
self._ctx.set_default_verify_paths()
def set_ciphers(self, ciphers: bytes | str) -> None:
if isinstance(ciphers, str):
ciphers = ciphers.encode("utf-8")
self._ctx.set_cipher_list(ciphers)
def load_verify_locations(
self,
cafile: str | None = None,
capath: str | None = None,
cadata: bytes | None = None,
) -> None:
if cafile is not None:
cafile = cafile.encode("utf-8") # type: ignore[assignment]
if capath is not None:
capath = capath.encode("utf-8") # type: ignore[assignment]
try:
self._ctx.load_verify_locations(cafile, capath)
if cadata is not None:
self._ctx.load_verify_locations(BytesIO(cadata))
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"unable to load trusted certificates: {e!r}") from e
def load_cert_chain(
self,
certfile: str,
keyfile: str | None = None,
password: str | None = None,
) -> None:
try:
self._ctx.use_certificate_chain_file(certfile)
if password is not None:
if not isinstance(password, bytes):
password = password.encode("utf-8") # type: ignore[assignment]
self._ctx.set_passwd_cb(lambda *_: password)
self._ctx.use_privatekey_file(keyfile or certfile)
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"Unable to load certificate chain: {e!r}") from e
def set_alpn_protocols(self, protocols: list[bytes | str]) -> None:
protocols = [util.util.to_bytes(p, "ascii") for p in protocols]
return self._ctx.set_alpn_protos(protocols) # type: ignore[no-any-return]
def wrap_socket(
self,
sock: socket_cls,
server_side: bool = False,
do_handshake_on_connect: bool = True,
suppress_ragged_eofs: bool = True,
server_hostname: bytes | str | None = None,
) -> WrappedSocket:
cnx = OpenSSL.SSL.Connection(self._ctx, sock)
# If server_hostname is an IP, don't use it for SNI, per RFC6066 Section 3
if server_hostname and not util.ssl_.is_ipaddress(server_hostname):
if isinstance(server_hostname, str):
server_hostname = server_hostname.encode("utf-8")
cnx.set_tlsext_host_name(server_hostname)
cnx.set_connect_state()
while True:
try:
cnx.do_handshake()
except OpenSSL.SSL.WantReadError as e:
if not util.wait_for_read(sock, sock.gettimeout()):
raise TimeoutError("select timed out") from e
continue
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"bad handshake: {e!r}") from e
break
return WrappedSocket(cnx, sock)
def _set_ctx_options(self) -> None:
self._ctx.set_options(
self._options
| _openssl_to_ssl_minimum_version[self._minimum_version]
| _openssl_to_ssl_maximum_version[self._maximum_version]
)
@property
def minimum_version(self) -> int:
return self._minimum_version
@minimum_version.setter
def minimum_version(self, minimum_version: int) -> None:
self._minimum_version = minimum_version
self._set_ctx_options()
@property
def maximum_version(self) -> int:
return self._maximum_version
@maximum_version.setter
def maximum_version(self, maximum_version: int) -> None:
self._maximum_version = maximum_version
self._set_ctx_options()
def _verify_callback(
cnx: OpenSSL.SSL.Connection,
x509: X509,
err_no: int,
err_depth: int,
return_code: int,
) -> bool:
return err_no == 0
@@ -0,0 +1,228 @@
"""
This module contains provisional support for SOCKS proxies from within
urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and
SOCKS5. To enable its functionality, either install PySocks or install this
module with the ``socks`` extra.
The SOCKS implementation supports the full range of urllib3 features. It also
supports the following SOCKS features:
- SOCKS4A (``proxy_url='socks4a://...``)
- SOCKS4 (``proxy_url='socks4://...``)
- SOCKS5 with remote DNS (``proxy_url='socks5h://...``)
- SOCKS5 with local DNS (``proxy_url='socks5://...``)
- Usernames and passwords for the SOCKS proxy
.. note::
It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in
your ``proxy_url`` to ensure that DNS resolution is done from the remote
server instead of client-side when connecting to a domain name.
SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5
supports IPv4, IPv6, and domain names.
When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url``
will be sent as the ``userid`` section of the SOCKS request:
.. code-block:: python
proxy_url="socks4a://<userid>@proxy-host"
When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion
of the ``proxy_url`` will be sent as the username/password to authenticate
with the proxy:
.. code-block:: python
proxy_url="socks5h://<username>:<password>@proxy-host"
"""
from __future__ import annotations
try:
import socks # type: ignore[import-untyped]
except ImportError:
import warnings
from ..exceptions import DependencyWarning
warnings.warn(
(
"SOCKS support in urllib3 requires the installation of optional "
"dependencies: specifically, PySocks. For more information, see "
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies"
),
DependencyWarning,
)
raise
import typing
from socket import timeout as SocketTimeout
from ..connection import HTTPConnection, HTTPSConnection
from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from ..exceptions import ConnectTimeoutError, NewConnectionError
from ..poolmanager import PoolManager
from ..util.url import parse_url
try:
import ssl
except ImportError:
ssl = None # type: ignore[assignment]
class _TYPE_SOCKS_OPTIONS(typing.TypedDict):
socks_version: int
proxy_host: str | None
proxy_port: str | None
username: str | None
password: str | None
rdns: bool
class SOCKSConnection(HTTPConnection):
"""
A plain-text HTTP connection that connects via a SOCKS proxy.
"""
def __init__(
self,
_socks_options: _TYPE_SOCKS_OPTIONS,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
self._socks_options = _socks_options
super().__init__(*args, **kwargs)
def _new_conn(self) -> socks.socksocket:
"""
Establish a new connection via the SOCKS proxy.
"""
extra_kw: dict[str, typing.Any] = {}
if self.source_address:
extra_kw["source_address"] = self.source_address
if self.socket_options:
extra_kw["socket_options"] = self.socket_options
try:
conn = socks.create_connection(
(self.host, self.port),
proxy_type=self._socks_options["socks_version"],
proxy_addr=self._socks_options["proxy_host"],
proxy_port=self._socks_options["proxy_port"],
proxy_username=self._socks_options["username"],
proxy_password=self._socks_options["password"],
proxy_rdns=self._socks_options["rdns"],
timeout=self.timeout,
**extra_kw,
)
except SocketTimeout as e:
raise ConnectTimeoutError(
self,
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
) from e
except socks.ProxyError as e:
# This is fragile as hell, but it seems to be the only way to raise
# useful errors here.
if e.socket_err:
error = e.socket_err
if isinstance(error, SocketTimeout):
raise ConnectTimeoutError(
self,
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
) from e
else:
# Adding `from e` messes with coverage somehow, so it's omitted.
# See #2386.
raise NewConnectionError(
self, f"Failed to establish a new connection: {error}"
)
else: # Defensive: see https://github.com/urllib3/urllib3/pull/3728#pullrequestreview-3816302703
raise NewConnectionError(
self, f"Failed to establish a new connection: {e}"
) from e
except OSError as e: # Defensive: PySocks should catch all these.
raise NewConnectionError(
self, f"Failed to establish a new connection: {e}"
) from e
return conn
# We don't need to duplicate the Verified/Unverified distinction from
# urllib3/connection.py here because the HTTPSConnection will already have been
# correctly set to either the Verified or Unverified form by that module. This
# means the SOCKSHTTPSConnection will automatically be the correct type.
class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection):
pass
class SOCKSHTTPConnectionPool(HTTPConnectionPool):
ConnectionCls = SOCKSConnection
class SOCKSHTTPSConnectionPool(HTTPSConnectionPool):
ConnectionCls = SOCKSHTTPSConnection
class SOCKSProxyManager(PoolManager):
"""
A version of the urllib3 ProxyManager that routes connections via the
defined SOCKS proxy.
"""
pool_classes_by_scheme = {
"http": SOCKSHTTPConnectionPool,
"https": SOCKSHTTPSConnectionPool,
}
def __init__(
self,
proxy_url: str,
username: str | None = None,
password: str | None = None,
num_pools: int = 10,
headers: typing.Mapping[str, str] | None = None,
**connection_pool_kw: typing.Any,
):
parsed = parse_url(proxy_url)
if username is None and password is None and parsed.auth is not None:
split = parsed.auth.split(":")
if len(split) == 2:
username, password = split
if parsed.scheme == "socks5":
socks_version = socks.PROXY_TYPE_SOCKS5
rdns = False
elif parsed.scheme == "socks5h":
socks_version = socks.PROXY_TYPE_SOCKS5
rdns = True
elif parsed.scheme == "socks4":
socks_version = socks.PROXY_TYPE_SOCKS4
rdns = False
elif parsed.scheme == "socks4a":
socks_version = socks.PROXY_TYPE_SOCKS4
rdns = True
else:
raise ValueError(f"Unable to determine SOCKS version from {proxy_url}")
self.proxy_url = proxy_url
socks_options = {
"socks_version": socks_version,
"proxy_host": parsed.host,
"proxy_port": parsed.port,
"username": username,
"password": password,
"rdns": rdns,
}
connection_pool_kw["_socks_options"] = socks_options
super().__init__(num_pools, headers, **connection_pool_kw)
self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme
@@ -0,0 +1,335 @@
from __future__ import annotations
import socket
import typing
import warnings
from email.errors import MessageDefect
from http.client import IncompleteRead as httplib_IncompleteRead
if typing.TYPE_CHECKING:
from .connection import HTTPConnection
from .connectionpool import ConnectionPool
from .response import HTTPResponse
from .util.retry import Retry
# Base Exceptions
class HTTPError(Exception):
"""Base exception used by this module."""
class HTTPWarning(Warning):
"""Base warning used by this module."""
_TYPE_REDUCE_RESULT = tuple[typing.Callable[..., object], tuple[object, ...]]
class PoolError(HTTPError):
"""Base exception for errors caused within a pool."""
def __init__(self, pool: ConnectionPool, message: str) -> None:
self.pool = pool
self._message = message
super().__init__(f"{pool}: {message}")
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
# For pickling purposes.
return self.__class__, (None, self._message)
class RequestError(PoolError):
"""Base exception for PoolErrors that have associated URLs."""
def __init__(self, pool: ConnectionPool, url: str | None, message: str) -> None:
self.url = url
super().__init__(pool, message)
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
# For pickling purposes.
return self.__class__, (None, self.url, self._message)
class SSLError(HTTPError):
"""Raised when SSL certificate fails in an HTTPS connection."""
class ProxyError(HTTPError):
"""Raised when the connection to a proxy fails."""
# The original error is also available as __cause__.
original_error: Exception
def __init__(self, message: str, error: Exception) -> None:
super().__init__(message, error)
self.original_error = error
class DecodeError(HTTPError):
"""Raised when automatic decoding based on Content-Type fails."""
class ProtocolError(HTTPError):
"""Raised when something unexpected happens mid-request/response."""
#: Renamed to ProtocolError but aliased for backwards compatibility.
ConnectionError = ProtocolError
# Leaf Exceptions
class MaxRetryError(RequestError):
"""Raised when the maximum number of retries is exceeded.
:param pool: The connection pool
:type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
:param str url: The requested Url
:param reason: The underlying error
:type reason: :class:`Exception`
"""
def __init__(
self, pool: ConnectionPool, url: str | None, reason: Exception | None = None
) -> None:
self.reason = reason
message = f"Max retries exceeded with url: {url} (Caused by {reason!r})"
super().__init__(pool, url, message)
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
# For pickling purposes.
return self.__class__, (None, self.url, self.reason)
class HostChangedError(RequestError):
"""Raised when an existing pool gets a request for a foreign host."""
def __init__(
self, pool: ConnectionPool, url: str, retries: Retry | int = 3
) -> None:
message = f"Tried to open a foreign host with url: {url}"
super().__init__(pool, url, message)
self.retries = retries
class TimeoutStateError(HTTPError):
"""Raised when passing an invalid state to a timeout"""
class TimeoutError(HTTPError):
"""Raised when a socket timeout error occurs.
Catching this error will catch both :exc:`ReadTimeoutErrors
<ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
"""
class ReadTimeoutError(TimeoutError, RequestError):
"""Raised when a socket timeout occurs while receiving data from a server"""
# This timeout error does not have a URL attached and needs to inherit from the
# base HTTPError
class ConnectTimeoutError(TimeoutError):
"""Raised when a socket timeout occurs while connecting to a server"""
class NewConnectionError(ConnectTimeoutError, HTTPError):
"""Raised when we fail to establish a new connection. Usually ECONNREFUSED."""
def __init__(self, conn: HTTPConnection, message: str) -> None:
self.conn = conn
self._message = message
super().__init__(f"{conn}: {message}")
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
# For pickling purposes.
return self.__class__, (None, self._message)
@property
def pool(self) -> HTTPConnection:
warnings.warn(
"The 'pool' property is deprecated and will be removed "
"in urllib3 v3.0. Use 'conn' instead.",
FutureWarning,
stacklevel=2,
)
return self.conn
class NameResolutionError(NewConnectionError):
"""Raised when host name resolution fails."""
def __init__(self, host: str, conn: HTTPConnection, reason: socket.gaierror):
message = f"Failed to resolve '{host}' ({reason})"
self._host = host
self._reason = reason
super().__init__(conn, message)
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
# For pickling purposes.
return self.__class__, (self._host, None, self._reason)
class EmptyPoolError(PoolError):
"""Raised when a pool runs out of connections and no more are allowed."""
class FullPoolError(PoolError):
"""Raised when we try to add a connection to a full pool in blocking mode."""
class ClosedPoolError(PoolError):
"""Raised when a request enters a pool after the pool has been closed."""
class LocationValueError(ValueError, HTTPError):
"""Raised when there is something wrong with a given URL input."""
class LocationParseError(LocationValueError):
"""Raised when get_host or similar fails to parse the URL input."""
def __init__(self, location: str) -> None:
message = f"Failed to parse: {location}"
super().__init__(message)
self.location = location
class URLSchemeUnknown(LocationValueError):
"""Raised when a URL input has an unsupported scheme."""
def __init__(self, scheme: str):
message = f"Not supported URL scheme {scheme}"
super().__init__(message)
self.scheme = scheme
class ResponseError(HTTPError):
"""Used as a container for an error reason supplied in a MaxRetryError."""
GENERIC_ERROR = "too many error responses"
SPECIFIC_ERROR = "too many {status_code} error responses"
class SecurityWarning(HTTPWarning):
"""Warned when performing security reducing actions"""
class InsecureRequestWarning(SecurityWarning):
"""Warned when making an unverified HTTPS request."""
class NotOpenSSLWarning(SecurityWarning):
"""Warned when using unsupported SSL library"""
class SystemTimeWarning(SecurityWarning):
"""Warned when system time is suspected to be wrong"""
class InsecurePlatformWarning(SecurityWarning):
"""Warned when certain TLS/SSL configuration is not available on a platform."""
class DependencyWarning(HTTPWarning):
"""
Warned when an attempt is made to import a module with missing optional
dependencies.
"""
class ResponseNotChunked(ProtocolError, ValueError):
"""Response needs to be chunked in order to read it as chunks."""
class BodyNotHttplibCompatible(HTTPError):
"""
Body should be :class:`http.client.HTTPResponse` like
(have an fp attribute which returns raw chunks) for read_chunked().
"""
class IncompleteRead(HTTPError, httplib_IncompleteRead):
"""
Response length doesn't match expected Content-Length
Subclass of :class:`http.client.IncompleteRead` to allow int value
for ``partial`` to avoid creating large objects on streamed reads.
"""
partial: int # type: ignore[assignment]
expected: int
def __init__(self, partial: int, expected: int) -> None:
self.partial = partial
self.expected = expected
def __repr__(self) -> str:
return "IncompleteRead(%i bytes read, %i more expected)" % (
self.partial,
self.expected,
)
class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
"""Invalid chunk length in a chunked response."""
def __init__(self, response: HTTPResponse, length: bytes) -> None:
self.partial: int = response.tell() # type: ignore[assignment]
self.expected: int | None = response.length_remaining
self.response = response
self.length = length
def __repr__(self) -> str:
return "InvalidChunkLength(got length %r, %i bytes read)" % (
self.length,
self.partial,
)
class InvalidHeader(HTTPError):
"""The header provided was somehow invalid."""
class ProxySchemeUnknown(AssertionError, URLSchemeUnknown):
"""ProxyManager does not support the supplied scheme"""
# TODO(t-8ch): Stop inheriting from AssertionError in v2.0.
def __init__(self, scheme: str | None) -> None:
# 'localhost' is here because our URL parser parses
# localhost:8080 -> scheme=localhost, remove if we fix this.
if scheme == "localhost":
scheme = None
if scheme is None:
message = "Proxy URL had no scheme, should start with http:// or https://"
else:
message = f"Proxy URL had unsupported scheme {scheme}, should use http:// or https://"
super().__init__(message)
class ProxySchemeUnsupported(ValueError):
"""Fetching HTTPS resources through HTTPS proxies is unsupported"""
class HeaderParsingError(HTTPError):
"""Raised by assert_header_parsing, but we convert it to a log.warning statement."""
def __init__(
self, defects: list[MessageDefect], unparsed_data: bytes | str | None
) -> None:
message = f"{defects or 'Unknown'}, unparsed data: {unparsed_data!r}"
super().__init__(message)
class UnrewindableBodyError(HTTPError):
"""urllib3 encountered an error when trying to rewind a body"""
@@ -0,0 +1,341 @@
from __future__ import annotations
import email.utils
import mimetypes
import typing
_TYPE_FIELD_VALUE = typing.Union[str, bytes]
_TYPE_FIELD_VALUE_TUPLE = typing.Union[
_TYPE_FIELD_VALUE,
tuple[str, _TYPE_FIELD_VALUE],
tuple[str, _TYPE_FIELD_VALUE, str],
]
def guess_content_type(
filename: str | None, default: str = "application/octet-stream"
) -> str:
"""
Guess the "Content-Type" of a file.
:param filename:
The filename to guess the "Content-Type" of using :mod:`mimetypes`.
:param default:
If no "Content-Type" can be guessed, default to `default`.
"""
if filename:
return mimetypes.guess_type(filename)[0] or default
return default
def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
"""
Helper function to format and quote a single header parameter using the
strategy defined in RFC 2231.
Particularly useful for header parameters which might contain
non-ASCII values, like file names. This follows
`RFC 2388 Section 4.4 <https://tools.ietf.org/html/rfc2388#section-4.4>`_.
:param name:
The name of the parameter, a string expected to be ASCII only.
:param value:
The value of the parameter, provided as ``bytes`` or `str``.
:returns:
An RFC-2231-formatted unicode string.
.. deprecated:: 2.0.0
Will be removed in urllib3 v3.0. This is not valid for
``multipart/form-data`` header parameters.
"""
import warnings
warnings.warn(
"'format_header_param_rfc2231' is insecure, deprecated and will be "
"removed in urllib3 v3.0. This is not valid for "
"multipart/form-data header parameters.",
FutureWarning,
stacklevel=2,
)
if isinstance(value, bytes):
value = value.decode("utf-8")
if not any(ch in value for ch in '"\\\r\n'):
result = f'{name}="{value}"'
try:
result.encode("ascii")
except (UnicodeEncodeError, UnicodeDecodeError):
pass
else:
return result
value = email.utils.encode_rfc2231(value, "utf-8")
value = f"{name}*={value}"
return value
def format_multipart_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
"""
Format and quote a single multipart header parameter.
This follows the `WHATWG HTML Standard`_ as of 2021/06/10, matching
the behavior of current browser and curl versions. Values are
assumed to be UTF-8. The ``\\n``, ``\\r``, and ``"`` characters are
percent encoded.
.. _WHATWG HTML Standard:
https://html.spec.whatwg.org/multipage/
form-control-infrastructure.html#multipart-form-data
:param name:
The name of the parameter, an ASCII-only ``str``.
:param value:
The value of the parameter, a ``str`` or UTF-8 encoded
``bytes``.
:returns:
A string ``name="value"`` with the escaped value.
.. versionchanged:: 2.0.0
Matches the WHATWG HTML Standard as of 2021/06/10. Control
characters are no longer percent encoded.
.. versionchanged:: 2.0.0
Renamed from ``format_header_param_html5`` and
``format_header_param``. The old names will be removed in
urllib3 v3.0.
"""
if isinstance(value, bytes):
value = value.decode("utf-8")
# percent encode \n \r "
value = value.translate({10: "%0A", 13: "%0D", 34: "%22"})
return f'{name}="{value}"'
def format_header_param_html5(name: str, value: _TYPE_FIELD_VALUE) -> str:
"""
.. deprecated:: 2.0.0
Renamed to :func:`format_multipart_header_param`. Will be
removed in urllib3 v3.0.
"""
import warnings
warnings.warn(
"'format_header_param_html5' has been renamed to "
"'format_multipart_header_param'. The old name will be "
"removed in urllib3 v3.0.",
FutureWarning,
stacklevel=2,
)
return format_multipart_header_param(name, value)
def format_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
"""
.. deprecated:: 2.0.0
Renamed to :func:`format_multipart_header_param`. Will be
removed in urllib3 v3.0.
"""
import warnings
warnings.warn(
"'format_header_param' has been renamed to "
"'format_multipart_header_param'. The old name will be "
"removed in urllib3 v3.0.",
FutureWarning,
stacklevel=2,
)
return format_multipart_header_param(name, value)
class RequestField:
"""
A data container for request body parameters.
:param name:
The name of this request field. Must be unicode.
:param data:
The data/value body.
:param filename:
An optional filename of the request field. Must be unicode.
:param headers:
An optional dict-like object of headers to initially use for the field.
.. versionchanged:: 2.0.0
The ``header_formatter`` parameter is deprecated and will
be removed in urllib3 v3.0.
"""
def __init__(
self,
name: str,
data: _TYPE_FIELD_VALUE,
filename: str | None = None,
headers: typing.Mapping[str, str] | None = None,
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
):
self._name = name
self._filename = filename
self.data = data
self.headers: dict[str, str | None] = {}
if headers:
self.headers = dict(headers)
if header_formatter is not None:
import warnings
warnings.warn(
"The 'header_formatter' parameter is deprecated and "
"will be removed in urllib3 v3.0.",
FutureWarning,
stacklevel=2,
)
self.header_formatter = header_formatter
else:
self.header_formatter = format_multipart_header_param
@classmethod
def from_tuples(
cls,
fieldname: str,
value: _TYPE_FIELD_VALUE_TUPLE,
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
) -> RequestField:
"""
A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
Supports constructing :class:`~urllib3.fields.RequestField` from
parameter of key/value strings AND key/filetuple. A filetuple is a
(filename, data, MIME type) tuple where the MIME type is optional.
For example::
'foo': 'bar',
'fakefile': ('foofile.txt', 'contents of foofile'),
'realfile': ('barfile.txt', open('realfile').read()),
'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'),
'nonamefile': 'contents of nonamefile field',
Field names and filenames must be unicode.
"""
filename: str | None
content_type: str | None
data: _TYPE_FIELD_VALUE
if isinstance(value, tuple):
if len(value) == 3:
filename, data, content_type = value
else:
filename, data = value
content_type = guess_content_type(filename)
else:
filename = None
content_type = None
data = value
request_param = cls(
fieldname, data, filename=filename, header_formatter=header_formatter
)
request_param.make_multipart(content_type=content_type)
return request_param
def _render_part(self, name: str, value: _TYPE_FIELD_VALUE) -> str:
"""
Override this method to change how each multipart header
parameter is formatted. By default, this calls
:func:`format_multipart_header_param`.
:param name:
The name of the parameter, an ASCII-only ``str``.
:param value:
The value of the parameter, a ``str`` or UTF-8 encoded
``bytes``.
:meta public:
"""
return self.header_formatter(name, value)
def _render_parts(
self,
header_parts: (
dict[str, _TYPE_FIELD_VALUE | None]
| typing.Sequence[tuple[str, _TYPE_FIELD_VALUE | None]]
),
) -> str:
"""
Helper function to format and quote a single header.
Useful for single headers that are composed of multiple items. E.g.,
'Content-Disposition' fields.
:param header_parts:
A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format
as `k1="v1"; k2="v2"; ...`.
"""
iterable: typing.Iterable[tuple[str, _TYPE_FIELD_VALUE | None]]
parts = []
if isinstance(header_parts, dict):
iterable = header_parts.items()
else:
iterable = header_parts
for name, value in iterable:
if value is not None:
parts.append(self._render_part(name, value))
return "; ".join(parts)
def render_headers(self) -> str:
"""
Renders the headers for this request field.
"""
lines = []
sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
for sort_key in sort_keys:
if self.headers.get(sort_key, False):
lines.append(f"{sort_key}: {self.headers[sort_key]}")
for header_name, header_value in self.headers.items():
if header_name not in sort_keys:
if header_value:
lines.append(f"{header_name}: {header_value}")
lines.append("\r\n")
return "\r\n".join(lines)
def make_multipart(
self,
content_disposition: str | None = None,
content_type: str | None = None,
content_location: str | None = None,
) -> None:
"""
Makes this request field into a multipart request field.
This method overrides "Content-Disposition", "Content-Type" and
"Content-Location" headers to the request parameter.
:param content_disposition:
The 'Content-Disposition' of the request body. Defaults to 'form-data'
:param content_type:
The 'Content-Type' of the request body.
:param content_location:
The 'Content-Location' of the request body.
"""
content_disposition = (content_disposition or "form-data") + "; ".join(
[
"",
self._render_parts(
(("name", self._name), ("filename", self._filename))
),
]
)
self.headers["Content-Disposition"] = content_disposition
self.headers["Content-Type"] = content_type
self.headers["Content-Location"] = content_location
@@ -0,0 +1,89 @@
from __future__ import annotations
import binascii
import codecs
import os
import typing
from io import BytesIO
from .fields import _TYPE_FIELD_VALUE_TUPLE, RequestField
writer = codecs.lookup("utf-8")[3]
_TYPE_FIELDS_SEQUENCE = typing.Sequence[
typing.Union[tuple[str, _TYPE_FIELD_VALUE_TUPLE], RequestField]
]
_TYPE_FIELDS = typing.Union[
_TYPE_FIELDS_SEQUENCE,
typing.Mapping[str, _TYPE_FIELD_VALUE_TUPLE],
]
def choose_boundary() -> str:
"""
Our embarrassingly-simple replacement for mimetools.choose_boundary.
"""
return binascii.hexlify(os.urandom(16)).decode()
def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]:
"""
Iterate over fields.
Supports list of (k, v) tuples and dicts, and lists of
:class:`~urllib3.fields.RequestField`.
"""
iterable: typing.Iterable[RequestField | tuple[str, _TYPE_FIELD_VALUE_TUPLE]]
if isinstance(fields, typing.Mapping):
iterable = fields.items()
else:
iterable = fields
for field in iterable:
if isinstance(field, RequestField):
yield field
else:
yield RequestField.from_tuples(*field)
def encode_multipart_formdata(
fields: _TYPE_FIELDS, boundary: str | None = None
) -> tuple[bytes, str]:
"""
Encode a dictionary of ``fields`` using the multipart/form-data MIME format.
:param fields:
Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`).
Values are processed by :func:`urllib3.fields.RequestField.from_tuples`.
:param boundary:
If not specified, then a random boundary will be generated using
:func:`urllib3.filepost.choose_boundary`.
"""
body = BytesIO()
if boundary is None:
boundary = choose_boundary()
for field in iter_field_objects(fields):
body.write(f"--{boundary}\r\n".encode("latin-1"))
writer(body).write(field.render_headers())
data = field.data
if isinstance(data, int):
data = str(data) # Backwards compatibility
if isinstance(data, str):
writer(body).write(data)
else:
body.write(data)
body.write(b"\r\n")
body.write(f"--{boundary}--\r\n".encode("latin-1"))
content_type = f"multipart/form-data; boundary={boundary}"
return body.getvalue(), content_type
@@ -0,0 +1,53 @@
from __future__ import annotations
from importlib.metadata import version
__all__ = [
"inject_into_urllib3",
"extract_from_urllib3",
]
import typing
orig_HTTPSConnection: typing.Any = None
def inject_into_urllib3() -> None:
# First check if h2 version is valid
h2_version = version("h2")
if not h2_version.startswith("4."):
raise ImportError(
"urllib3 v2 supports h2 version 4.x.x, currently "
f"the 'h2' module is compiled with {h2_version!r}. "
"See: https://github.com/urllib3/urllib3/issues/3290"
)
# Import here to avoid circular dependencies.
from .. import connection as urllib3_connection
from .. import util as urllib3_util
from ..connectionpool import HTTPSConnectionPool
from ..util import ssl_ as urllib3_util_ssl
from .connection import HTTP2Connection
global orig_HTTPSConnection
orig_HTTPSConnection = urllib3_connection.HTTPSConnection
HTTPSConnectionPool.ConnectionCls = HTTP2Connection
urllib3_connection.HTTPSConnection = HTTP2Connection # type: ignore[misc]
# TODO: Offer 'http/1.1' as well, but for testing purposes this is handy.
urllib3_util.ALPN_PROTOCOLS = ["h2"]
urllib3_util_ssl.ALPN_PROTOCOLS = ["h2"]
def extract_from_urllib3() -> None:
from .. import connection as urllib3_connection
from .. import util as urllib3_util
from ..connectionpool import HTTPSConnectionPool
from ..util import ssl_ as urllib3_util_ssl
HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection
urllib3_connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc]
urllib3_util.ALPN_PROTOCOLS = ["http/1.1"]
urllib3_util_ssl.ALPN_PROTOCOLS = ["http/1.1"]
@@ -0,0 +1,356 @@
from __future__ import annotations
import logging
import re
import threading
import types
import typing
import h2.config
import h2.connection
import h2.events
from .._base_connection import _TYPE_BODY
from .._collections import HTTPHeaderDict
from ..connection import HTTPSConnection, _get_default_user_agent
from ..exceptions import ConnectionError
from ..response import BaseHTTPResponse
orig_HTTPSConnection = HTTPSConnection
T = typing.TypeVar("T")
log = logging.getLogger(__name__)
RE_IS_LEGAL_HEADER_NAME = re.compile(rb"^[!#$%&'*+\-.^_`|~0-9a-z]+$")
RE_IS_ILLEGAL_HEADER_VALUE = re.compile(rb"[\0\x00\x0a\x0d\r\n]|^[ \r\n\t]|[ \r\n\t]$")
def _is_legal_header_name(name: bytes) -> bool:
"""
"An implementation that validates fields according to the definitions in Sections
5.1 and 5.5 of [HTTP] only needs an additional check that field names do not
include uppercase characters." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
`http.client._is_legal_header_name` does not validate the field name according to the
HTTP 1.1 spec, so we do that here, in addition to checking for uppercase characters.
This does not allow for the `:` character in the header name, so should not
be used to validate pseudo-headers.
"""
return bool(RE_IS_LEGAL_HEADER_NAME.match(name))
def _is_illegal_header_value(value: bytes) -> bool:
"""
"A field value MUST NOT contain the zero value (ASCII NUL, 0x00), line feed
(ASCII LF, 0x0a), or carriage return (ASCII CR, 0x0d) at any position. A field
value MUST NOT start or end with an ASCII whitespace character (ASCII SP or HTAB,
0x20 or 0x09)." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
"""
return bool(RE_IS_ILLEGAL_HEADER_VALUE.search(value))
class _LockedObject(typing.Generic[T]):
"""
A wrapper class that hides a specific object behind a lock.
The goal here is to provide a simple way to protect access to an object
that cannot safely be simultaneously accessed from multiple threads. The
intended use of this class is simple: take hold of it with a context
manager, which returns the protected object.
"""
__slots__ = (
"lock",
"_obj",
)
def __init__(self, obj: T):
self.lock = threading.RLock()
self._obj = obj
def __enter__(self) -> T:
self.lock.acquire()
return self._obj
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: types.TracebackType | None,
) -> None:
self.lock.release()
class HTTP2Connection(HTTPSConnection):
def __init__(
self, host: str, port: int | None = None, **kwargs: typing.Any
) -> None:
self._h2_conn = self._new_h2_conn()
self._h2_stream: int | None = None
self._headers: list[tuple[bytes, bytes]] = []
if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive:
raise NotImplementedError("Proxies aren't supported with HTTP/2")
super().__init__(host, port, **kwargs)
if self._tunnel_host is not None:
raise NotImplementedError("Tunneling isn't supported with HTTP/2")
def _new_h2_conn(self) -> _LockedObject[h2.connection.H2Connection]:
config = h2.config.H2Configuration(client_side=True)
return _LockedObject(h2.connection.H2Connection(config=config))
def connect(self) -> None:
super().connect()
with self._h2_conn as conn:
conn.initiate_connection()
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
def putrequest( # type: ignore[override]
self,
method: str,
url: str,
**kwargs: typing.Any,
) -> None:
"""putrequest
This deviates from the HTTPConnection method signature since we never need to override
sending accept-encoding headers or the host header.
"""
if "skip_host" in kwargs:
raise NotImplementedError("`skip_host` isn't supported")
if "skip_accept_encoding" in kwargs:
raise NotImplementedError("`skip_accept_encoding` isn't supported")
self._request_url = url or "/"
self._validate_path(url) # type: ignore[attr-defined]
if ":" in self.host:
authority = f"[{self.host}]:{self.port or 443}"
else:
authority = f"{self.host}:{self.port or 443}"
self._headers.append((b":scheme", b"https"))
self._headers.append((b":method", method.encode()))
self._headers.append((b":authority", authority.encode()))
self._headers.append((b":path", url.encode()))
with self._h2_conn as conn:
self._h2_stream = conn.get_next_available_stream_id()
def putheader(self, header: str | bytes, *values: str | bytes) -> None: # type: ignore[override]
# TODO SKIPPABLE_HEADERS from urllib3 are ignored.
header = header.encode() if isinstance(header, str) else header
header = header.lower() # A lot of upstream code uses capitalized headers.
if not _is_legal_header_name(header):
raise ValueError(f"Illegal header name {str(header)}")
for value in values:
value = value.encode() if isinstance(value, str) else value
if _is_illegal_header_value(value):
raise ValueError(f"Illegal header value {str(value)}")
self._headers.append((header, value))
def endheaders(self, message_body: typing.Any = None) -> None: # type: ignore[override]
if self._h2_stream is None:
raise ConnectionError("Must call `putrequest` first.")
with self._h2_conn as conn:
conn.send_headers(
stream_id=self._h2_stream,
headers=self._headers,
end_stream=(message_body is None),
)
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
self._headers = [] # Reset headers for the next request.
def send(self, data: typing.Any) -> None:
"""Send data to the server.
`data` can be: `str`, `bytes`, an iterable, or file-like objects
that support a .read() method.
"""
if self._h2_stream is None:
raise ConnectionError("Must call `putrequest` first.")
with self._h2_conn as conn:
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
if hasattr(data, "read"): # file-like objects
while True:
chunk = data.read(self.blocksize)
if not chunk:
break
if isinstance(chunk, str):
chunk = chunk.encode()
conn.send_data(self._h2_stream, chunk, end_stream=False)
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
conn.end_stream(self._h2_stream)
return
if isinstance(data, str): # str -> bytes
data = data.encode()
try:
if isinstance(data, bytes):
conn.send_data(self._h2_stream, data, end_stream=True)
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
else:
for chunk in data:
conn.send_data(self._h2_stream, chunk, end_stream=False)
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
conn.end_stream(self._h2_stream)
except TypeError:
raise TypeError(
"`data` should be str, bytes, iterable, or file. got %r"
% type(data)
)
def set_tunnel(
self,
host: str,
port: int | None = None,
headers: typing.Mapping[str, str] | None = None,
scheme: str = "http",
) -> None:
raise NotImplementedError(
"HTTP/2 does not support setting up a tunnel through a proxy"
)
def getresponse( # type: ignore[override]
self,
) -> HTTP2Response:
status = None
data = bytearray()
with self._h2_conn as conn:
end_stream = False
while not end_stream:
# TODO: Arbitrary read value.
if received_data := self.sock.recv(65535):
events = conn.receive_data(received_data)
for event in events:
if isinstance(event, h2.events.ResponseReceived):
headers = HTTPHeaderDict()
for header, value in event.headers:
if header == b":status":
status = int(value.decode())
else:
headers.add(
header.decode("ascii"), value.decode("ascii")
)
elif isinstance(event, h2.events.DataReceived):
data += event.data
conn.acknowledge_received_data(
event.flow_controlled_length, event.stream_id
)
elif isinstance(event, h2.events.StreamEnded):
end_stream = True
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
assert status is not None
return HTTP2Response(
status=status,
headers=headers,
request_url=self._request_url,
data=bytes(data),
)
def request( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
*,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
**kwargs: typing.Any,
) -> None:
"""Send an HTTP/2 request"""
if "chunked" in kwargs:
# TODO this is often present from upstream.
# raise NotImplementedError("`chunked` isn't supported with HTTP/2")
pass
if self.sock is not None:
self.sock.settimeout(self.timeout)
self.putrequest(method, url)
headers = headers or {}
for k, v in headers.items():
if k.lower() == "transfer-encoding" and v == "chunked":
continue
else:
self.putheader(k, v)
if b"user-agent" not in dict(self._headers):
self.putheader(b"user-agent", _get_default_user_agent())
if body:
self.endheaders(message_body=body)
self.send(body)
else:
self.endheaders()
def close(self) -> None:
with self._h2_conn as conn:
try:
conn.close_connection()
if data := conn.data_to_send():
self.sock.sendall(data)
except Exception:
pass
# Reset all our HTTP/2 connection state.
self._h2_conn = self._new_h2_conn()
self._h2_stream = None
self._headers = []
super().close()
class HTTP2Response(BaseHTTPResponse):
# TODO: This is a woefully incomplete response object, but works for non-streaming.
def __init__(
self,
status: int,
headers: HTTPHeaderDict,
request_url: str,
data: bytes,
decode_content: bool = False, # TODO: support decoding
) -> None:
super().__init__(
status=status,
headers=headers,
# Following CPython, we map HTTP versions to major * 10 + minor integers
version=20,
version_string="HTTP/2",
# No reason phrase in HTTP/2
reason=None,
decode_content=decode_content,
request_url=request_url,
)
self._data = data
self.length_remaining = 0
@property
def data(self) -> bytes:
return self._data
def get_redirect_location(self) -> None:
return None
def close(self) -> None:
pass
@@ -0,0 +1,87 @@
from __future__ import annotations
import threading
class _HTTP2ProbeCache:
__slots__ = (
"_lock",
"_cache_locks",
"_cache_values",
)
def __init__(self) -> None:
self._lock = threading.Lock()
self._cache_locks: dict[tuple[str, int], threading.RLock] = {}
self._cache_values: dict[tuple[str, int], bool | None] = {}
def acquire_and_get(self, host: str, port: int) -> bool | None:
# By the end of this block we know that
# _cache_[values,locks] is available.
value = None
with self._lock:
key = (host, port)
try:
value = self._cache_values[key]
# If it's a known value we return right away.
if value is not None:
return value
except KeyError:
self._cache_locks[key] = threading.RLock()
self._cache_values[key] = None
# If the value is unknown, we acquire the lock to signal
# to the requesting thread that the probe is in progress
# or that the current thread needs to return their findings.
key_lock = self._cache_locks[key]
key_lock.acquire()
try:
# If the by the time we get the lock the value has been
# updated we want to return the updated value.
value = self._cache_values[key]
# In case an exception like KeyboardInterrupt is raised here.
except BaseException as e: # Defensive:
assert not isinstance(e, KeyError) # KeyError shouldn't be possible.
key_lock.release()
raise
return value
def set_and_release(
self, host: str, port: int, supports_http2: bool | None
) -> None:
key = (host, port)
key_lock = self._cache_locks[key]
with key_lock: # Uses an RLock, so can be locked again from same thread.
if supports_http2 is None and self._cache_values[key] is not None:
raise ValueError(
"Cannot reset HTTP/2 support for origin after value has been set."
) # Defensive: not expected in normal usage
self._cache_values[key] = supports_http2
key_lock.release()
def _values(self) -> dict[tuple[str, int], bool | None]:
"""This function is for testing purposes only. Gets the current state of the probe cache"""
with self._lock:
return {k: v for k, v in self._cache_values.items()}
def _reset(self) -> None:
"""This function is for testing purposes only. Reset the cache values"""
with self._lock:
self._cache_locks = {}
self._cache_values = {}
_HTTP2_PROBE_CACHE = _HTTP2ProbeCache()
set_and_release = _HTTP2_PROBE_CACHE.set_and_release
acquire_and_get = _HTTP2_PROBE_CACHE.acquire_and_get
_values = _HTTP2_PROBE_CACHE._values
_reset = _HTTP2_PROBE_CACHE._reset
__all__ = [
"set_and_release",
"acquire_and_get",
]
@@ -0,0 +1,653 @@
from __future__ import annotations
import functools
import logging
import typing
import warnings
from types import TracebackType
from urllib.parse import urljoin
from ._collections import HTTPHeaderDict, RecentlyUsedContainer
from ._request_methods import RequestMethods
from .connection import ProxyConfig
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
from .exceptions import (
LocationValueError,
MaxRetryError,
ProxySchemeUnknown,
URLSchemeUnknown,
)
from .response import BaseHTTPResponse
from .util.connection import _TYPE_SOCKET_OPTIONS
from .util.proxy import connection_requires_http_tunnel
from .util.retry import Retry
from .util.timeout import Timeout
from .util.url import Url, parse_url
if typing.TYPE_CHECKING:
import ssl
from typing_extensions import Self
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
log = logging.getLogger(__name__)
SSL_KEYWORDS = (
"key_file",
"cert_file",
"cert_reqs",
"ca_certs",
"ca_cert_data",
"ssl_version",
"ssl_minimum_version",
"ssl_maximum_version",
"ca_cert_dir",
"ssl_context",
"key_password",
"server_hostname",
)
# Default value for `blocksize` - a new parameter introduced to
# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
_DEFAULT_BLOCKSIZE = 16384
class PoolKey(typing.NamedTuple):
"""
All known keyword arguments that could be provided to the pool manager, its
pools, or the underlying connections.
All custom key schemes should include the fields in this key at a minimum.
"""
key_scheme: str
key_host: str
key_port: int | None
key_timeout: Timeout | float | int | None
key_retries: Retry | bool | int | None
key_block: bool | None
key_source_address: tuple[str, int] | None
key_key_file: str | None
key_key_password: str | None
key_cert_file: str | None
key_cert_reqs: str | None
key_ca_certs: str | None
key_ca_cert_data: str | bytes | None
key_ssl_version: int | str | None
key_ssl_minimum_version: ssl.TLSVersion | None
key_ssl_maximum_version: ssl.TLSVersion | None
key_ca_cert_dir: str | None
key_ssl_context: ssl.SSLContext | None
key_maxsize: int | None
key_headers: frozenset[tuple[str, str]] | None
key__proxy: Url | None
key__proxy_headers: frozenset[tuple[str, str]] | None
key__proxy_config: ProxyConfig | None
key_socket_options: _TYPE_SOCKET_OPTIONS | None
key__socks_options: frozenset[tuple[str, str]] | None
key_assert_hostname: bool | str | None
key_assert_fingerprint: str | None
key_server_hostname: str | None
key_blocksize: int | None
def _default_key_normalizer(
key_class: type[PoolKey], request_context: dict[str, typing.Any]
) -> PoolKey:
"""
Create a pool key out of a request context dictionary.
According to RFC 3986, both the scheme and host are case-insensitive.
Therefore, this function normalizes both before constructing the pool
key for an HTTPS request. If you wish to change this behaviour, provide
alternate callables to ``key_fn_by_scheme``.
:param key_class:
The class to use when constructing the key. This should be a namedtuple
with the ``scheme`` and ``host`` keys at a minimum.
:type key_class: namedtuple
:param request_context:
A dictionary-like object that contain the context for a request.
:type request_context: dict
:return: A namedtuple that can be used as a connection pool key.
:rtype: PoolKey
"""
# Since we mutate the dictionary, make a copy first
context = request_context.copy()
context["scheme"] = context["scheme"].lower()
context["host"] = context["host"].lower()
# These are both dictionaries and need to be transformed into frozensets
for key in ("headers", "_proxy_headers", "_socks_options"):
if key in context and context[key] is not None:
context[key] = frozenset(context[key].items())
# The socket_options key may be a list and needs to be transformed into a
# tuple.
socket_opts = context.get("socket_options")
if socket_opts is not None:
context["socket_options"] = tuple(socket_opts)
# Map the kwargs to the names in the namedtuple - this is necessary since
# namedtuples can't have fields starting with '_'.
for key in list(context.keys()):
context["key_" + key] = context.pop(key)
# Default to ``None`` for keys missing from the context
for field in key_class._fields:
if field not in context:
context[field] = None
# Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
if context.get("key_blocksize") is None:
context["key_blocksize"] = _DEFAULT_BLOCKSIZE
return key_class(**context)
#: A dictionary that maps a scheme to a callable that creates a pool key.
#: This can be used to alter the way pool keys are constructed, if desired.
#: Each PoolManager makes a copy of this dictionary so they can be configured
#: globally here, or individually on the instance.
key_fn_by_scheme = {
"http": functools.partial(_default_key_normalizer, PoolKey),
"https": functools.partial(_default_key_normalizer, PoolKey),
}
pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
class PoolManager(RequestMethods):
"""
Allows for arbitrary requests while transparently keeping track of
necessary connection pools for you.
:param num_pools:
Number of connection pools to cache before discarding the least
recently used pool.
:param headers:
Headers to include with all requests, unless other headers are given
explicitly.
:param \\**connection_pool_kw:
Additional parameters are used to create fresh
:class:`urllib3.connectionpool.ConnectionPool` instances.
Example:
.. code-block:: python
import urllib3
http = urllib3.PoolManager(num_pools=2)
resp1 = http.request("GET", "https://google.com/")
resp2 = http.request("GET", "https://google.com/mail")
resp3 = http.request("GET", "https://yahoo.com/")
print(len(http.pools))
# 2
"""
proxy: Url | None = None
proxy_config: ProxyConfig | None = None
def __init__(
self,
num_pools: int = 10,
headers: typing.Mapping[str, str] | None = None,
**connection_pool_kw: typing.Any,
) -> None:
super().__init__(headers)
# PoolManager handles redirects itself in PoolManager.urlopen().
# It always passes redirect=False to the underlying connection pool to
# suppress per-pool redirect handling. If the user supplied a non-Retry
# value (int/bool/etc) for retries and we let the pool normalize it
# while redirect=False, the resulting Retry object would have redirect
# handling disabled, which can interfere with PoolManager's own
# redirect logic. Normalize here so redirects remain governed solely by
# PoolManager logic.
if "retries" in connection_pool_kw:
retries = connection_pool_kw["retries"]
if not isinstance(retries, Retry):
retries = Retry.from_int(retries)
connection_pool_kw = connection_pool_kw.copy()
connection_pool_kw["retries"] = retries
self.connection_pool_kw = connection_pool_kw
self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
self.pools = RecentlyUsedContainer(num_pools)
# Locally set the pool classes and keys so other PoolManagers can
# override them.
self.pool_classes_by_scheme = pool_classes_by_scheme
self.key_fn_by_scheme = key_fn_by_scheme.copy()
def __enter__(self) -> Self:
return self
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> typing.Literal[False]:
self.clear()
# Return False to re-raise any potential exceptions
return False
def _new_pool(
self,
scheme: str,
host: str,
port: int,
request_context: dict[str, typing.Any] | None = None,
) -> HTTPConnectionPool:
"""
Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
any additional pool keyword arguments.
If ``request_context`` is provided, it is provided as keyword arguments
to the pool class used. This method is used to actually create the
connection pools handed out by :meth:`connection_from_url` and
companion methods. It is intended to be overridden for customization.
"""
pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
if request_context is None:
request_context = self.connection_pool_kw.copy()
# Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
# set to 'None' in the request_context.
if request_context.get("blocksize") is None:
request_context["blocksize"] = _DEFAULT_BLOCKSIZE
# Although the context has everything necessary to create the pool,
# this function has historically only used the scheme, host, and port
# in the positional args. When an API change is acceptable these can
# be removed.
for key in ("scheme", "host", "port"):
request_context.pop(key, None)
if scheme == "http":
for kw in SSL_KEYWORDS:
request_context.pop(kw, None)
return pool_cls(host, port, **request_context)
def clear(self) -> None:
"""
Empty our store of pools and direct them all to close.
This will not affect in-flight connections, but they will not be
re-used after completion.
"""
self.pools.clear()
def connection_from_host(
self,
host: str | None,
port: int | None = None,
scheme: str | None = "http",
pool_kwargs: dict[str, typing.Any] | None = None,
) -> HTTPConnectionPool:
"""
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
If ``port`` isn't given, it will be derived from the ``scheme`` using
``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
provided, it is merged with the instance's ``connection_pool_kw``
variable and used to create the new connection pool, if one is
needed.
"""
if not host:
raise LocationValueError("No host specified.")
request_context = self._merge_pool_kwargs(pool_kwargs)
request_context["scheme"] = scheme or "http"
if not port:
port = port_by_scheme.get(request_context["scheme"].lower(), 80)
request_context["port"] = port
request_context["host"] = host
return self.connection_from_context(request_context)
def connection_from_context(
self, request_context: dict[str, typing.Any]
) -> HTTPConnectionPool:
"""
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
``request_context`` must at least contain the ``scheme`` key and its
value must be a key in ``key_fn_by_scheme`` instance variable.
"""
if "strict" in request_context:
warnings.warn(
"The 'strict' parameter is no longer needed on Python 3+. "
"This will raise an error in urllib3 v3.0.",
FutureWarning,
)
request_context.pop("strict")
scheme = request_context["scheme"].lower()
pool_key_constructor = self.key_fn_by_scheme.get(scheme)
if not pool_key_constructor:
raise URLSchemeUnknown(scheme)
pool_key = pool_key_constructor(request_context)
return self.connection_from_pool_key(pool_key, request_context=request_context)
def connection_from_pool_key(
self, pool_key: PoolKey, request_context: dict[str, typing.Any]
) -> HTTPConnectionPool:
"""
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
``pool_key`` should be a namedtuple that only contains immutable
objects. At a minimum it must have the ``scheme``, ``host``, and
``port`` fields.
"""
with self.pools.lock:
# If the scheme, host, or port doesn't match existing open
# connections, open a new ConnectionPool.
pool = self.pools.get(pool_key)
if pool:
return pool
# Make a fresh ConnectionPool of the desired type
scheme = request_context["scheme"]
host = request_context["host"]
port = request_context["port"]
pool = self._new_pool(scheme, host, port, request_context=request_context)
self.pools[pool_key] = pool
return pool
def connection_from_url(
self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
) -> HTTPConnectionPool:
"""
Similar to :func:`urllib3.connectionpool.connection_from_url`.
If ``pool_kwargs`` is not provided and a new pool needs to be
constructed, ``self.connection_pool_kw`` is used to initialize
the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
is provided, it is used instead. Note that if a new pool does not
need to be created for the request, the provided ``pool_kwargs`` are
not used.
"""
u = parse_url(url)
return self.connection_from_host(
u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
)
def _merge_pool_kwargs(
self, override: dict[str, typing.Any] | None
) -> dict[str, typing.Any]:
"""
Merge a dictionary of override values for self.connection_pool_kw.
This does not modify self.connection_pool_kw and returns a new dict.
Any keys in the override dictionary with a value of ``None`` are
removed from the merged dictionary.
"""
base_pool_kwargs = self.connection_pool_kw.copy()
if override:
for key, value in override.items():
if value is None:
try:
del base_pool_kwargs[key]
except KeyError:
pass
else:
base_pool_kwargs[key] = value
return base_pool_kwargs
def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
"""
Indicates if the proxy requires the complete destination URL in the
request. Normally this is only needed when not using an HTTP CONNECT
tunnel.
"""
if self.proxy is None:
return False
return not connection_requires_http_tunnel(
self.proxy, self.proxy_config, parsed_url.scheme
)
def urlopen( # type: ignore[override]
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
) -> BaseHTTPResponse:
"""
Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
with custom cross-host redirect logic and only sends the request-uri
portion of the ``url``.
The given ``url`` parameter must be absolute, such that an appropriate
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
u = parse_url(url)
if u.scheme is None:
warnings.warn(
"URLs without a scheme (ie 'https://') are deprecated and will raise an error "
"in urllib3 v3.0. To avoid this FutureWarning ensure all URLs "
"start with 'https://' or 'http://'. Read more in this issue: "
"https://github.com/urllib3/urllib3/issues/2920",
category=FutureWarning,
stacklevel=2,
)
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
kw["assert_same_host"] = False
kw["redirect"] = False
if "headers" not in kw:
kw["headers"] = self.headers
if self._proxy_requires_url_absolute_form(u):
response = conn.urlopen(method, url, **kw)
else:
response = conn.urlopen(method, u.request_uri, **kw)
redirect_location = redirect and response.get_redirect_location()
if not redirect_location:
return response
# Support relative URLs for redirecting.
redirect_location = urljoin(url, redirect_location)
if response.status == 303:
# Change the method according to RFC 9110, Section 15.4.4.
method = "GET"
# And lose the body not to transfer anything sensitive.
kw["body"] = None
kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
retries = kw.get("retries", response.retries)
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect)
# Strip headers marked as unsafe to forward to the redirected location.
# Check remove_headers_on_redirect to avoid a potential network call within
# conn.is_same_host() which may use socket.gethostbyname() in the future.
if retries.remove_headers_on_redirect and not conn.is_same_host(
redirect_location
):
new_headers = kw["headers"].copy()
for header in kw["headers"]:
if header.lower() in retries.remove_headers_on_redirect:
new_headers.pop(header, None)
kw["headers"] = new_headers
try:
retries = retries.increment(method, url, response=response, _pool=conn)
except MaxRetryError:
if retries.raise_on_redirect:
response.drain_conn()
raise
return response
kw["retries"] = retries
kw["redirect"] = redirect
log.info("Redirecting %s -> %s", url, redirect_location)
response.drain_conn()
return self.urlopen(method, redirect_location, **kw)
class ProxyManager(PoolManager):
"""
Behaves just like :class:`PoolManager`, but sends all requests through
the defined proxy, using the CONNECT method for HTTPS URLs.
:param proxy_url:
The URL of the proxy to be used.
:param proxy_headers:
A dictionary containing headers that will be sent to the proxy. In case
of HTTP they are being sent with each request, while in the
HTTPS/CONNECT case they are sent only once. Could be used for proxy
authentication.
:param proxy_ssl_context:
The proxy SSL context is used to establish the TLS connection to the
proxy when using HTTPS proxies.
:param use_forwarding_for_https:
(Defaults to False) If set to True will forward requests to the HTTPS
proxy to be made on behalf of the client instead of creating a TLS
tunnel via the CONNECT method. **Enabling this flag means that request
and response headers and content will be visible from the HTTPS proxy**
whereas tunneling keeps request and response headers and content
private. IP address, target hostname, SNI, and port are always visible
to an HTTPS proxy even when this flag is disabled.
:param proxy_assert_hostname:
The hostname of the certificate to verify against.
:param proxy_assert_fingerprint:
The fingerprint of the certificate to verify against.
Example:
.. code-block:: python
import urllib3
proxy = urllib3.ProxyManager("https://localhost:3128/")
resp1 = proxy.request("GET", "http://google.com/")
resp2 = proxy.request("GET", "http://httpbin.org/")
# One pool was shared by both plain HTTP requests.
print(len(proxy.pools))
# 1
resp3 = proxy.request("GET", "https://httpbin.org/")
resp4 = proxy.request("GET", "https://twitter.com/")
# A separate pool was added for each HTTPS target.
print(len(proxy.pools))
# 3
"""
def __init__(
self,
proxy_url: str,
num_pools: int = 10,
headers: typing.Mapping[str, str] | None = None,
proxy_headers: typing.Mapping[str, str] | None = None,
proxy_ssl_context: ssl.SSLContext | None = None,
use_forwarding_for_https: bool = False,
proxy_assert_hostname: None | str | typing.Literal[False] = None,
proxy_assert_fingerprint: str | None = None,
**connection_pool_kw: typing.Any,
) -> None:
if isinstance(proxy_url, HTTPConnectionPool):
str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
else:
str_proxy_url = proxy_url
proxy = parse_url(str_proxy_url)
if proxy.scheme not in ("http", "https"):
raise ProxySchemeUnknown(proxy.scheme)
if not proxy.port:
port = port_by_scheme.get(proxy.scheme, 80)
proxy = proxy._replace(port=port)
self.proxy = proxy
self.proxy_headers = proxy_headers or {}
self.proxy_ssl_context = proxy_ssl_context
self.proxy_config = ProxyConfig(
proxy_ssl_context,
use_forwarding_for_https,
proxy_assert_hostname,
proxy_assert_fingerprint,
)
connection_pool_kw["_proxy"] = self.proxy
connection_pool_kw["_proxy_headers"] = self.proxy_headers
connection_pool_kw["_proxy_config"] = self.proxy_config
super().__init__(num_pools, headers, **connection_pool_kw)
def connection_from_host(
self,
host: str | None,
port: int | None = None,
scheme: str | None = "http",
pool_kwargs: dict[str, typing.Any] | None = None,
) -> HTTPConnectionPool:
if scheme == "https":
return super().connection_from_host(
host, port, scheme, pool_kwargs=pool_kwargs
)
return super().connection_from_host(
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
)
def _set_proxy_headers(
self, url: str, headers: typing.Mapping[str, str] | None = None
) -> typing.Mapping[str, str]:
"""
Sets headers needed by proxies: specifically, the Accept and Host
headers. Only sets headers not provided by the user.
"""
headers_ = {"Accept": "*/*"}
netloc = parse_url(url).netloc
if netloc:
headers_["Host"] = netloc
if headers:
headers_.update(headers)
return headers_
def urlopen( # type: ignore[override]
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
) -> BaseHTTPResponse:
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
u = parse_url(url)
if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
# For connections using HTTP CONNECT, httplib sets the necessary
# headers on the CONNECT to the proxy. If we're not using CONNECT,
# we'll definitely need to set 'Host' at the very least.
headers = kw.get("headers", self.headers)
kw["headers"] = self._set_proxy_headers(url, headers)
return super().urlopen(method, url, redirect=redirect, **kw)
def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
return ProxyManager(proxy_url=url, **kw)
@@ -0,0 +1,42 @@
# For backwards compatibility, provide imports that used to be here.
from __future__ import annotations
from .connection import is_connection_dropped
from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers
from .response import is_fp_closed
from .retry import Retry
from .ssl_ import (
ALPN_PROTOCOLS,
IS_PYOPENSSL,
SSLContext,
assert_fingerprint,
create_urllib3_context,
resolve_cert_reqs,
resolve_ssl_version,
ssl_wrap_socket,
)
from .timeout import Timeout
from .url import Url, parse_url
from .wait import wait_for_read, wait_for_write
__all__ = (
"IS_PYOPENSSL",
"SSLContext",
"ALPN_PROTOCOLS",
"Retry",
"Timeout",
"Url",
"assert_fingerprint",
"create_urllib3_context",
"is_connection_dropped",
"is_fp_closed",
"parse_url",
"make_headers",
"resolve_cert_reqs",
"resolve_ssl_version",
"ssl_wrap_socket",
"wait_for_read",
"wait_for_write",
"SKIP_HEADER",
"SKIPPABLE_HEADERS",
)
@@ -0,0 +1,137 @@
from __future__ import annotations
import socket
import typing
from ..exceptions import LocationParseError
from .timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
_TYPE_SOCKET_OPTIONS = list[tuple[int, int, typing.Union[int, bytes]]]
if typing.TYPE_CHECKING:
from .._base_connection import BaseHTTPConnection
def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specific
"""
Returns True if the connection is dropped and should be closed.
:param conn: :class:`urllib3.connection.HTTPConnection` object.
"""
return not conn.is_connected
# This function is copied from socket.py in the Python 2.7 standard
# library test suite. Added to its signature is only `socket_options`.
# One additional modification is that we avoid binding to IPv6 servers
# discovered in DNS if the system doesn't have IPv6 functionality.
def create_connection(
address: tuple[str, int],
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
socket_options: _TYPE_SOCKET_OPTIONS | None = None,
) -> socket.socket:
"""Connect to *address* and return the socket object.
Convenience function. Connect to *address* (a 2-tuple ``(host,
port)``) and return the socket object. Passing the optional
*timeout* parameter will set the timeout on the socket instance
before attempting to connect. If no *timeout* is supplied, the
global default timeout setting returned by :func:`socket.getdefaulttimeout`
is used. If *source_address* is set it must be a tuple of (host, port)
for the socket to bind as a source address before making the connection.
An host of '' or port 0 tells the OS to use the default.
"""
host, port = address
if host.startswith("["):
host = host.strip("[]")
err = None
# Using the value from allowed_gai_family() in the context of getaddrinfo lets
# us select whether to work with IPv4 DNS records, IPv6 records, or both.
# The original create_connection function always returns all records.
family = allowed_gai_family()
try:
host.encode("idna")
except UnicodeError:
raise LocationParseError(f"'{host}', label empty or too long") from None
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
sock = None
try:
sock = socket.socket(af, socktype, proto)
# If provided, set socket level options before connecting.
_set_socket_options(sock, socket_options)
if timeout is not _DEFAULT_TIMEOUT:
sock.settimeout(timeout)
if source_address:
sock.bind(source_address)
sock.connect(sa)
# Break explicitly a reference cycle
err = None
return sock
except OSError as _:
err = _
if sock is not None:
sock.close()
if err is not None:
try:
raise err
finally:
# Break explicitly a reference cycle
err = None
else:
raise OSError("getaddrinfo returns an empty list")
def _set_socket_options(
sock: socket.socket, options: _TYPE_SOCKET_OPTIONS | None
) -> None:
if options is None:
return
for opt in options:
sock.setsockopt(*opt)
def allowed_gai_family() -> socket.AddressFamily:
"""This function is designed to work in the context of
getaddrinfo, where family=socket.AF_UNSPEC is the default and
will perform a DNS search for both IPv6 and IPv4 records."""
family = socket.AF_INET
if HAS_IPV6:
family = socket.AF_UNSPEC
return family
def _has_ipv6(host: str) -> bool:
"""Returns True if the system can bind an IPv6 address."""
sock = None
has_ipv6 = False
if socket.has_ipv6:
# has_ipv6 returns true if cPython was compiled with IPv6 support.
# It does not tell us if the system has IPv6 support enabled. To
# determine that we must bind to an IPv6 address.
# https://github.com/urllib3/urllib3/pull/611
# https://bugs.python.org/issue658327
try:
sock = socket.socket(socket.AF_INET6)
sock.bind((host, 0))
has_ipv6 = True
except Exception:
pass
if sock:
sock.close()
return has_ipv6
HAS_IPV6 = _has_ipv6("::1")
@@ -0,0 +1,43 @@
from __future__ import annotations
import typing
from .url import Url
if typing.TYPE_CHECKING:
from ..connection import ProxyConfig
def connection_requires_http_tunnel(
proxy_url: Url | None = None,
proxy_config: ProxyConfig | None = None,
destination_scheme: str | None = None,
) -> bool:
"""
Returns True if the connection requires an HTTP CONNECT through the proxy.
:param URL proxy_url:
URL of the proxy.
:param ProxyConfig proxy_config:
Proxy configuration from poolmanager.py
:param str destination_scheme:
The scheme of the destination. (i.e https, http, etc)
"""
# If we're not using a proxy, no way to use a tunnel.
if proxy_url is None:
return False
# HTTP destinations never require tunneling, we always forward.
if destination_scheme == "http":
return False
# Support for forwarding with HTTPS proxies and HTTPS destinations.
if (
proxy_url.scheme == "https"
and proxy_config
and proxy_config.use_forwarding_for_https
):
return False
# Otherwise always use a tunnel.
return True
@@ -0,0 +1,263 @@
from __future__ import annotations
import io
import sys
import typing
from base64 import b64encode
from enum import Enum
from ..exceptions import UnrewindableBodyError
from .util import to_bytes
if typing.TYPE_CHECKING:
from typing import Final
# Pass as a value within ``headers`` to skip
# emitting some HTTP headers that are added automatically.
# The only headers that are supported are ``Accept-Encoding``,
# ``Host``, and ``User-Agent``.
SKIP_HEADER = "@@@SKIP_HEADER@@@"
SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"])
ACCEPT_ENCODING = "gzip,deflate"
try:
try:
import brotlicffi as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401
except ImportError:
import brotli as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401
except ImportError:
pass
else:
ACCEPT_ENCODING += ",br"
try:
if sys.version_info >= (3, 14):
from compression import zstd as _unused_module_zstd # noqa: F401
else:
from backports import zstd as _unused_module_zstd # noqa: F401
except ImportError:
pass
else:
ACCEPT_ENCODING += ",zstd"
class _TYPE_FAILEDTELL(Enum):
token = 0
_FAILEDTELL: Final[_TYPE_FAILEDTELL] = _TYPE_FAILEDTELL.token
_TYPE_BODY_POSITION = typing.Union[int, _TYPE_FAILEDTELL]
# When sending a request with these methods we aren't expecting
# a body so don't need to set an explicit 'Content-Length: 0'
# The reason we do this in the negative instead of tracking methods
# which 'should' have a body is because unknown methods should be
# treated as if they were 'POST' which *does* expect a body.
_METHODS_NOT_EXPECTING_BODY = {"GET", "HEAD", "DELETE", "TRACE", "OPTIONS", "CONNECT"}
def make_headers(
keep_alive: bool | None = None,
accept_encoding: bool | list[str] | str | None = None,
user_agent: str | None = None,
basic_auth: str | None = None,
proxy_basic_auth: str | None = None,
disable_cache: bool | None = None,
) -> dict[str, str]:
"""
Shortcuts for generating request headers.
:param keep_alive:
If ``True``, adds 'connection: keep-alive' header.
:param accept_encoding:
Can be a boolean, list, or string.
``True`` translates to 'gzip,deflate'. If the dependencies for
Brotli (either the ``brotli`` or ``brotlicffi`` package) and/or
Zstandard (the ``backports.zstd`` package for Python before 3.14)
algorithms are installed, then their encodings are
included in the string ('br' and 'zstd', respectively).
List will get joined by comma.
String will be used as provided.
:param user_agent:
String representing the user-agent you want, such as
"python-urllib3/0.6"
:param basic_auth:
Colon-separated username:password string for 'authorization: basic ...'
auth header.
:param proxy_basic_auth:
Colon-separated username:password string for 'proxy-authorization: basic ...'
auth header.
:param disable_cache:
If ``True``, adds 'cache-control: no-cache' header.
Example:
.. code-block:: python
import urllib3
print(urllib3.util.make_headers(keep_alive=True, user_agent="Batman/1.0"))
# {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
print(urllib3.util.make_headers(accept_encoding=True))
# {'accept-encoding': 'gzip,deflate'}
"""
headers: dict[str, str] = {}
if accept_encoding:
if isinstance(accept_encoding, str):
pass
elif isinstance(accept_encoding, list):
accept_encoding = ",".join(accept_encoding)
else:
accept_encoding = ACCEPT_ENCODING
headers["accept-encoding"] = accept_encoding
if user_agent:
headers["user-agent"] = user_agent
if keep_alive:
headers["connection"] = "keep-alive"
if basic_auth:
headers["authorization"] = (
f"Basic {b64encode(basic_auth.encode('latin-1')).decode()}"
)
if proxy_basic_auth:
headers["proxy-authorization"] = (
f"Basic {b64encode(proxy_basic_auth.encode('latin-1')).decode()}"
)
if disable_cache:
headers["cache-control"] = "no-cache"
return headers
def set_file_position(
body: typing.Any, pos: _TYPE_BODY_POSITION | None
) -> _TYPE_BODY_POSITION | None:
"""
If a position is provided, move file to that point.
Otherwise, we'll attempt to record a position for future use.
"""
if pos is not None:
rewind_body(body, pos)
elif getattr(body, "tell", None) is not None:
try:
pos = body.tell()
except OSError:
# This differentiates from None, allowing us to catch
# a failed `tell()` later when trying to rewind the body.
pos = _FAILEDTELL
return pos
def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -> None:
"""
Attempt to rewind body to a certain position.
Primarily used for request redirects and retries.
:param body:
File-like object that supports seek.
:param int pos:
Position to seek to in file.
"""
body_seek = getattr(body, "seek", None)
if body_seek is not None and isinstance(body_pos, int):
try:
body_seek(body_pos)
except OSError as e:
raise UnrewindableBodyError(
"An error occurred when rewinding request body for redirect/retry."
) from e
elif body_pos is _FAILEDTELL:
raise UnrewindableBodyError(
"Unable to record file position for rewinding "
"request body during a redirect/retry."
)
else:
raise ValueError(
f"body_pos must be of type integer, instead it was {type(body_pos)}."
)
class ChunksAndContentLength(typing.NamedTuple):
chunks: typing.Iterable[bytes] | None
content_length: int | None
def body_to_chunks(
body: typing.Any | None, method: str, blocksize: int
) -> ChunksAndContentLength:
"""Takes the HTTP request method, body, and blocksize and
transforms them into an iterable of chunks to pass to
socket.sendall() and an optional 'Content-Length' header.
A 'Content-Length' of 'None' indicates the length of the body
can't be determined so should use 'Transfer-Encoding: chunked'
for framing instead.
"""
chunks: typing.Iterable[bytes] | None
content_length: int | None
# No body, we need to make a recommendation on 'Content-Length'
# based on whether that request method is expected to have
# a body or not.
if body is None:
chunks = None
if method.upper() not in _METHODS_NOT_EXPECTING_BODY:
content_length = 0
else:
content_length = None
# Bytes or strings become bytes
elif isinstance(body, (str, bytes)):
chunks = (to_bytes(body),)
content_length = len(chunks[0])
# File-like object, TODO: use seek() and tell() for length?
elif hasattr(body, "read"):
def chunk_readable() -> typing.Iterable[bytes]:
encode = isinstance(body, io.TextIOBase)
while True:
datablock = body.read(blocksize)
if not datablock:
break
if encode:
datablock = datablock.encode("utf-8")
yield datablock
chunks = chunk_readable()
content_length = None
# Otherwise we need to start checking via duck-typing.
else:
try:
# Check if the body implements the buffer API.
mv = memoryview(body)
except TypeError:
try:
# Check if the body is an iterable
chunks = iter(body)
content_length = None
except TypeError:
raise TypeError(
f"'body' must be a bytes-like object, file-like "
f"object, or iterable. Instead was {body!r}"
) from None
else:
# Since it implements the buffer API can be passed directly to socket.sendall()
chunks = (body,)
content_length = mv.nbytes
return ChunksAndContentLength(chunks=chunks, content_length=content_length)
@@ -0,0 +1,101 @@
from __future__ import annotations
import http.client as httplib
from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect
from ..exceptions import HeaderParsingError
def is_fp_closed(obj: object) -> bool:
"""
Checks whether a given file-like object is closed.
:param obj:
The file-like object to check.
"""
try:
# Check `isclosed()` first, in case Python3 doesn't set `closed`.
# GH Issue #928
return obj.isclosed() # type: ignore[no-any-return, attr-defined]
except AttributeError:
pass
try:
# Check via the official file-like-object way.
return obj.closed # type: ignore[no-any-return, attr-defined]
except AttributeError:
pass
try:
# Check if the object is a container for another file-like object that
# gets released on exhaustion (e.g. HTTPResponse).
return obj.fp is None # type: ignore[attr-defined]
except AttributeError:
pass
raise ValueError("Unable to determine whether fp is closed.")
def assert_header_parsing(headers: httplib.HTTPMessage) -> None:
"""
Asserts whether all headers have been successfully parsed.
Extracts encountered errors from the result of parsing headers.
Only works on Python 3.
:param http.client.HTTPMessage headers: Headers to verify.
:raises urllib3.exceptions.HeaderParsingError:
If parsing errors are found.
"""
# This will fail silently if we pass in the wrong kind of parameter.
# To make debugging easier add an explicit check.
if not isinstance(headers, httplib.HTTPMessage):
raise TypeError(f"expected httplib.Message, got {type(headers)}.")
unparsed_data = None
# get_payload is actually email.message.Message.get_payload;
# we're only interested in the result if it's not a multipart message
if not headers.is_multipart():
payload = headers.get_payload()
if isinstance(payload, (bytes, str)):
unparsed_data = payload
# httplib is assuming a response body is available
# when parsing headers even when httplib only sends
# header data to parse_headers() This results in
# defects on multipart responses in particular.
# See: https://github.com/urllib3/urllib3/issues/800
# So we ignore the following defects:
# - StartBoundaryNotFoundDefect:
# The claimed start boundary was never found.
# - MultipartInvariantViolationDefect:
# A message claimed to be a multipart but no subparts were found.
defects = [
defect
for defect in headers.defects
if not isinstance(
defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect)
)
]
if defects or unparsed_data:
raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)
def is_response_to_head(response: httplib.HTTPResponse) -> bool:
"""
Checks whether the request of a response has been a HEAD-request.
:param http.client.HTTPResponse response:
Response to check if the originating request
used 'HEAD' as a method.
"""
# FIXME: Can we do this somehow without accessing private httplib _method?
method_str = response._method # type: str # type: ignore[attr-defined]
return method_str.upper() == "HEAD"
@@ -0,0 +1,557 @@
from __future__ import annotations
import email
import logging
import random
import re
import time
import typing
from itertools import takewhile
from types import TracebackType
from ..exceptions import (
ConnectTimeoutError,
InvalidHeader,
MaxRetryError,
ProtocolError,
ProxyError,
ReadTimeoutError,
ResponseError,
)
from .util import reraise
if typing.TYPE_CHECKING:
from typing_extensions import Self
from ..connectionpool import ConnectionPool
from ..response import BaseHTTPResponse
log = logging.getLogger(__name__)
# Data structure for representing the metadata of requests that result in a retry.
class RequestHistory(typing.NamedTuple):
method: str | None
url: str | None
error: Exception | None
status: int | None
redirect_location: str | None
class Retry:
"""Retry configuration.
Each retry attempt will create a new Retry object with updated values, so
they can be safely reused.
Retries can be defined as a default for a pool:
.. code-block:: python
retries = Retry(connect=5, read=2, redirect=5)
http = PoolManager(retries=retries)
response = http.request("GET", "https://example.com/")
Or per-request (which overrides the default for the pool):
.. code-block:: python
response = http.request("GET", "https://example.com/", retries=Retry(10))
Retries can be disabled by passing ``False``:
.. code-block:: python
response = http.request("GET", "https://example.com/", retries=False)
Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
retries are disabled, in which case the causing exception will be raised.
:param int total:
Total number of retries to allow. Takes precedence over other counts.
Set to ``None`` to remove this constraint and fall back on other
counts.
Set to ``0`` to fail on the first retry.
Set to ``False`` to disable and imply ``raise_on_redirect=False``.
:param int connect:
How many connection-related errors to retry on.
These are errors raised before the request is sent to the remote server,
which we assume has not triggered the server to process the request.
Set to ``0`` to fail on the first retry of this type.
:param int read:
How many times to retry on read errors.
These errors are raised after the request was sent to the server, so the
request may have side-effects.
Set to ``0`` to fail on the first retry of this type.
:param int redirect:
How many redirects to perform. Limit this to avoid infinite redirect
loops.
A redirect is a HTTP response with a status code 301, 302, 303, 307 or
308.
Set to ``0`` to fail on the first retry of this type.
Set to ``False`` to disable and imply ``raise_on_redirect=False``.
:param int status:
How many times to retry on bad status codes.
These are retries made on responses, where status code matches
``status_forcelist``.
Set to ``0`` to fail on the first retry of this type.
:param int other:
How many times to retry on other errors.
Other errors are errors that are not connect, read, redirect or status errors.
These errors might be raised after the request was sent to the server, so the
request might have side-effects.
Set to ``0`` to fail on the first retry of this type.
If ``total`` is not set, it's a good idea to set this to 0 to account
for unexpected edge cases and avoid infinite retry loops.
:param Collection allowed_methods:
Set of uppercased HTTP method verbs that we should retry on.
By default, we only retry on methods which are considered to be
idempotent (multiple requests with the same parameters end with the
same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
Set to a ``None`` value to retry on any verb.
:param Collection status_forcelist:
A set of integer HTTP status codes that we should force a retry on.
A retry is initiated if the request method is in ``allowed_methods``
and the response status code is in ``status_forcelist``.
By default, this is disabled with ``None``.
:param float backoff_factor:
A backoff factor to apply between attempts after the second try
(most errors are resolved immediately by a second try without a
delay). urllib3 will sleep for::
{backoff factor} * (2 ** ({number of previous retries}))
seconds. If `backoff_jitter` is non-zero, this sleep is extended by::
random.uniform(0, {backoff jitter})
seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will
sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever
be longer than `backoff_max`.
By default, backoff is disabled (factor set to 0).
:param float backoff_max:
The maximum backoff time (in seconds) between retry attempts.
This value caps the computed backoff from `backoff_factor`.
:param float backoff_jitter:
Random jitter amount (in seconds) added to the computed backoff.
Jitter is sampled uniformly from `0` to `backoff_jitter`.
:param bool raise_on_redirect: Whether, if the number of redirects is
exhausted, to raise a MaxRetryError, or to return a response with a
response code in the 3xx range.
:param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
whether we should raise an exception, or return a response,
if status falls in ``status_forcelist`` range and retries have
been exhausted.
:param tuple history: The history of the request encountered during
each call to :meth:`~Retry.increment`. The list is in the order
the requests occurred. Each list item is of class :class:`RequestHistory`.
:param bool respect_retry_after_header:
Whether to respect Retry-After header on status codes defined as
:attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
:param Collection remove_headers_on_redirect:
Sequence of headers to remove from the request when a response
indicating a redirect is returned before firing off the redirected
request.
:param int retry_after_max: Number of seconds to allow as the maximum for
Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`.
Any Retry-After headers larger than this value will be limited to this
value.
"""
#: Default methods to be used for ``allowed_methods``
DEFAULT_ALLOWED_METHODS = frozenset(
["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
)
#: Default status codes to be used for ``status_forcelist``
RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
#: Default headers to be used for ``remove_headers_on_redirect``
DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(
["Cookie", "Authorization", "Proxy-Authorization"]
)
#: Default maximum backoff time.
DEFAULT_BACKOFF_MAX = 120
# This is undocumented in the RFC. Setting to 6 hours matches other popular libraries.
#: Default maximum allowed value for Retry-After headers in seconds
DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600
# Backward compatibility; assigned outside of the class.
DEFAULT: typing.ClassVar[Retry]
def __init__(
self,
total: bool | int | None = 10,
connect: int | None = None,
read: int | None = None,
redirect: bool | int | None = None,
status: int | None = None,
other: int | None = None,
allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
status_forcelist: typing.Collection[int] | None = None,
backoff_factor: float = 0,
backoff_max: float = DEFAULT_BACKOFF_MAX,
raise_on_redirect: bool = True,
raise_on_status: bool = True,
history: tuple[RequestHistory, ...] | None = None,
respect_retry_after_header: bool = True,
remove_headers_on_redirect: typing.Collection[
str
] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
backoff_jitter: float = 0.0,
retry_after_max: int = DEFAULT_RETRY_AFTER_MAX,
) -> None:
self.total = total
self.connect = connect
self.read = read
self.status = status
self.other = other
if redirect is False or total is False:
redirect = 0
raise_on_redirect = False
self.redirect = redirect
self.status_forcelist = status_forcelist or set()
self.allowed_methods = allowed_methods
self.backoff_factor = backoff_factor
self.backoff_max = backoff_max
self.retry_after_max = retry_after_max
self.raise_on_redirect = raise_on_redirect
self.raise_on_status = raise_on_status
self.history = history or ()
self.respect_retry_after_header = respect_retry_after_header
self.remove_headers_on_redirect = frozenset(
h.lower() for h in remove_headers_on_redirect
)
self.backoff_jitter = backoff_jitter
def new(self, **kw: typing.Any) -> Self:
params = dict(
total=self.total,
connect=self.connect,
read=self.read,
redirect=self.redirect,
status=self.status,
other=self.other,
allowed_methods=self.allowed_methods,
status_forcelist=self.status_forcelist,
backoff_factor=self.backoff_factor,
backoff_max=self.backoff_max,
retry_after_max=self.retry_after_max,
raise_on_redirect=self.raise_on_redirect,
raise_on_status=self.raise_on_status,
history=self.history,
remove_headers_on_redirect=self.remove_headers_on_redirect,
respect_retry_after_header=self.respect_retry_after_header,
backoff_jitter=self.backoff_jitter,
)
params.update(kw)
return type(self)(**params) # type: ignore[arg-type]
@classmethod
def from_int(
cls,
retries: Retry | bool | int | None,
redirect: bool | int | None = True,
default: Retry | bool | int | None = None,
) -> Retry:
"""Backwards-compatibility for the old retries format."""
if retries is None:
retries = default if default is not None else cls.DEFAULT
if isinstance(retries, Retry):
return retries
redirect = bool(redirect) and None
new_retries = cls(retries, redirect=redirect)
log.debug("Converted retries value: %r -> %r", retries, new_retries)
return new_retries
def get_backoff_time(self) -> float:
"""Formula for computing the current backoff
:rtype: float
"""
# We want to consider only the last consecutive errors sequence (Ignore redirects).
consecutive_errors_len = len(
list(
takewhile(lambda x: x.redirect_location is None, reversed(self.history))
)
)
if consecutive_errors_len <= 1:
return 0
backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
if self.backoff_jitter != 0.0:
backoff_value += random.random() * self.backoff_jitter
return float(max(0, min(self.backoff_max, backoff_value)))
def parse_retry_after(self, retry_after: str) -> float:
seconds: float
# Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
if re.match(r"^\s*[0-9]+\s*$", retry_after):
seconds = int(retry_after)
else:
retry_date_tuple = email.utils.parsedate_tz(retry_after)
if retry_date_tuple is None:
raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
retry_date = email.utils.mktime_tz(retry_date_tuple)
seconds = retry_date - time.time()
seconds = max(seconds, 0)
# Check the seconds do not exceed the specified maximum
if seconds > self.retry_after_max:
seconds = self.retry_after_max
return seconds
def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
"""Get the value of Retry-After in seconds."""
retry_after = response.headers.get("Retry-After")
if retry_after is None:
return None
return self.parse_retry_after(retry_after)
def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
retry_after = self.get_retry_after(response)
if retry_after:
time.sleep(retry_after)
return True
return False
def _sleep_backoff(self) -> None:
backoff = self.get_backoff_time()
if backoff <= 0:
return
time.sleep(backoff)
def sleep(self, response: BaseHTTPResponse | None = None) -> None:
"""Sleep between retry attempts.
This method will respect a server's ``Retry-After`` response header
and sleep the duration of the time requested. If that is not present, it
will use an exponential backoff. By default, the backoff factor is 0 and
this method will return immediately.
"""
if self.respect_retry_after_header and response:
slept = self.sleep_for_retry(response)
if slept:
return
self._sleep_backoff()
def _is_connection_error(self, err: Exception) -> bool:
"""Errors when we're fairly sure that the server did not receive the
request, so it should be safe to retry.
"""
if isinstance(err, ProxyError):
err = err.original_error
return isinstance(err, ConnectTimeoutError)
def _is_read_error(self, err: Exception) -> bool:
"""Errors that occur after the request has been started, so we should
assume that the server began processing it.
"""
return isinstance(err, (ReadTimeoutError, ProtocolError))
def _is_method_retryable(self, method: str) -> bool:
"""Checks if a given HTTP method should be retried upon, depending if
it is included in the allowed_methods
"""
if self.allowed_methods and method.upper() not in self.allowed_methods:
return False
return True
def is_retry(
self, method: str, status_code: int, has_retry_after: bool = False
) -> bool:
"""Is this method/status code retryable? (Based on allowlists and control
variables such as the number of total retries to allow, whether to
respect the Retry-After header, whether this header is present, and
whether the returned status code is on the list of status codes to
be retried upon on the presence of the aforementioned header)
"""
if not self._is_method_retryable(method):
return False
if self.status_forcelist and status_code in self.status_forcelist:
return True
return bool(
self.total
and self.respect_retry_after_header
and has_retry_after
and (status_code in self.RETRY_AFTER_STATUS_CODES)
)
def is_exhausted(self) -> bool:
"""Are we out of retries?"""
retry_counts = [
x
for x in (
self.total,
self.connect,
self.read,
self.redirect,
self.status,
self.other,
)
if x
]
if not retry_counts:
return False
return min(retry_counts) < 0
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
return new_retry
def __repr__(self) -> str:
return (
f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
f"read={self.read}, redirect={self.redirect}, status={self.status})"
)
# For backwards compatibility (equivalent to pre-v1.9):
Retry.DEFAULT = Retry(3)
@@ -0,0 +1,477 @@
from __future__ import annotations
import hashlib
import hmac
import os
import socket
import sys
import typing
import warnings
from binascii import unhexlify
from ..exceptions import ProxySchemeUnsupported, SSLError
from .url import _BRACELESS_IPV6_ADDRZ_RE, _IPV4_RE
SSLContext = None
SSLTransport = None
HAS_NEVER_CHECK_COMMON_NAME = False
IS_PYOPENSSL = False
ALPN_PROTOCOLS = ["http/1.1"]
_TYPE_VERSION_INFO = tuple[int, int, int, str, int]
# Maps the length of a digest to a possible hash function producing this digest
HASHFUNC_MAP = {
length: getattr(hashlib, algorithm, None)
for length, algorithm in ((32, "md5"), (40, "sha1"), (64, "sha256"))
}
def _is_has_never_check_common_name_reliable(
openssl_version: str,
) -> bool:
# As of May 2023, all released versions of LibreSSL fail to reject certificates with
# only common names, see https://github.com/urllib3/urllib3/pull/3024
is_openssl = openssl_version.startswith("OpenSSL ")
return is_openssl
if typing.TYPE_CHECKING:
from ssl import VerifyMode
from typing import TypedDict
from .ssltransport import SSLTransport as SSLTransportType
class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False):
subjectAltName: tuple[tuple[str, str], ...]
subject: tuple[tuple[tuple[str, str], ...], ...]
serialNumber: str
# Mapping from 'ssl.PROTOCOL_TLSX' to 'TLSVersion.X'
_SSL_VERSION_TO_TLS_VERSION: dict[int, int] = {}
try: # Do we have ssl at all?
import ssl
from ssl import ( # type: ignore[assignment]
CERT_REQUIRED,
HAS_NEVER_CHECK_COMMON_NAME,
OP_NO_COMPRESSION,
OP_NO_TICKET,
OPENSSL_VERSION,
PROTOCOL_TLS,
PROTOCOL_TLS_CLIENT,
VERIFY_X509_PARTIAL_CHAIN,
VERIFY_X509_STRICT,
OP_NO_SSLv2,
OP_NO_SSLv3,
SSLContext,
TLSVersion,
)
PROTOCOL_SSLv23 = PROTOCOL_TLS
# Setting SSLContext.hostname_checks_common_name = False didn't work with
# LibreSSL, check details in the used function.
if HAS_NEVER_CHECK_COMMON_NAME and not _is_has_never_check_common_name_reliable(
OPENSSL_VERSION,
): # Defensive:
HAS_NEVER_CHECK_COMMON_NAME = False
# Need to be careful here in case old TLS versions get
# removed in future 'ssl' module implementations.
for attr in ("TLSv1", "TLSv1_1", "TLSv1_2"):
try:
_SSL_VERSION_TO_TLS_VERSION[getattr(ssl, f"PROTOCOL_{attr}")] = getattr(
TLSVersion, attr
)
except AttributeError: # Defensive:
continue
from .ssltransport import SSLTransport # type: ignore[assignment]
except ImportError:
OP_NO_COMPRESSION = 0x20000 # type: ignore[assignment, misc]
OP_NO_TICKET = 0x4000 # type: ignore[assignment, misc]
OP_NO_SSLv2 = 0x1000000 # type: ignore[assignment, misc]
OP_NO_SSLv3 = 0x2000000 # type: ignore[assignment, misc]
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 # type: ignore[assignment, misc]
PROTOCOL_TLS_CLIENT = 16 # type: ignore[assignment, misc]
VERIFY_X509_PARTIAL_CHAIN = 0x80000 # type: ignore[assignment,misc]
VERIFY_X509_STRICT = 0x20 # type: ignore[assignment, misc]
_TYPE_PEER_CERT_RET = typing.Union["_TYPE_PEER_CERT_RET_DICT", bytes, None]
def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
"""
Checks if given fingerprint matches the supplied certificate.
:param cert:
Certificate as bytes object.
:param fingerprint:
Fingerprint as string of hexdigits, can be interspersed by colons.
"""
if cert is None:
raise SSLError("No certificate for the peer.")
fingerprint = fingerprint.replace(":", "").lower()
digest_length = len(fingerprint)
if digest_length not in HASHFUNC_MAP:
raise SSLError(f"Fingerprint of invalid length: {fingerprint}")
hashfunc = HASHFUNC_MAP.get(digest_length)
if hashfunc is None:
raise SSLError(
f"Hash function implementation unavailable for fingerprint length: {digest_length}"
)
# We need encode() here for py32; works on py2 and p33.
fingerprint_bytes = unhexlify(fingerprint.encode())
cert_digest = hashfunc(cert).digest()
if not hmac.compare_digest(cert_digest, fingerprint_bytes):
raise SSLError(
f'Fingerprints did not match. Expected "{fingerprint}", got "{cert_digest.hex()}"'
)
def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode:
"""
Resolves the argument to a numeric constant, which can be passed to
the wrap_socket function/method from the ssl module.
Defaults to :data:`ssl.CERT_REQUIRED`.
If given a string it is assumed to be the name of the constant in the
:mod:`ssl` module or its abbreviation.
(So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
If it's neither `None` nor a string we assume it is already the numeric
constant which can directly be passed to wrap_socket.
"""
if candidate is None:
return CERT_REQUIRED
if isinstance(candidate, str):
res = getattr(ssl, candidate, None)
if res is None:
res = getattr(ssl, "CERT_" + candidate)
return res # type: ignore[no-any-return]
return candidate # type: ignore[return-value]
def resolve_ssl_version(candidate: None | int | str) -> int:
"""
like resolve_cert_reqs
"""
if candidate is None:
return PROTOCOL_TLS
if isinstance(candidate, str):
res = getattr(ssl, candidate, None)
if res is None:
res = getattr(ssl, "PROTOCOL_" + candidate)
return typing.cast(int, res)
return candidate
def create_urllib3_context(
ssl_version: int | None = None,
cert_reqs: int | None = None,
options: int | None = None,
ciphers: str | None = None,
ssl_minimum_version: int | None = None,
ssl_maximum_version: int | None = None,
verify_flags: int | None = None,
) -> ssl.SSLContext:
"""Creates and configures an :class:`ssl.SSLContext` instance for use with urllib3.
:param ssl_version:
The desired protocol version to use. This will default to
PROTOCOL_SSLv23 which will negotiate the highest protocol that both
the server and your installation of OpenSSL support.
This parameter is deprecated instead use 'ssl_minimum_version'.
:param ssl_minimum_version:
The minimum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
:param ssl_maximum_version:
The maximum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
Not recommended to set to anything other than 'ssl.TLSVersion.MAXIMUM_SUPPORTED' which is the
default value.
:param cert_reqs:
Whether to require the certificate verification. This defaults to
``ssl.CERT_REQUIRED``.
:param options:
Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``.
:param ciphers:
Which cipher suites to allow the server to select. Defaults to either system configured
ciphers if OpenSSL 1.1.1+, otherwise uses a secure default set of ciphers.
:param verify_flags:
The flags for certificate verification operations. These default to
``ssl.VERIFY_X509_PARTIAL_CHAIN`` and ``ssl.VERIFY_X509_STRICT`` for Python 3.13+.
:returns:
Constructed SSLContext object with specified options
:rtype: SSLContext
"""
if SSLContext is None:
raise TypeError("Can't create an SSLContext object without an ssl module")
# This means 'ssl_version' was specified as an exact value.
if ssl_version not in (None, PROTOCOL_TLS, PROTOCOL_TLS_CLIENT):
# Disallow setting 'ssl_version' and 'ssl_minimum|maximum_version'
# to avoid conflicts.
if ssl_minimum_version is not None or ssl_maximum_version is not None:
raise ValueError(
"Can't specify both 'ssl_version' and either "
"'ssl_minimum_version' or 'ssl_maximum_version'"
)
# 'ssl_version' is deprecated and will be removed in the future.
else:
# Use 'ssl_minimum_version' and 'ssl_maximum_version' instead.
ssl_minimum_version = _SSL_VERSION_TO_TLS_VERSION.get(
ssl_version, TLSVersion.MINIMUM_SUPPORTED
)
ssl_maximum_version = _SSL_VERSION_TO_TLS_VERSION.get(
ssl_version, TLSVersion.MAXIMUM_SUPPORTED
)
# This warning message is pushing users to use 'ssl_minimum_version'
# instead of both min/max. Best practice is to only set the minimum version and
# keep the maximum version to be it's default value: 'TLSVersion.MAXIMUM_SUPPORTED'
warnings.warn(
"'ssl_version' option is deprecated and will be "
"removed in urllib3 v3.0. Instead use 'ssl_minimum_version'",
category=FutureWarning,
stacklevel=2,
)
context = SSLContext(PROTOCOL_TLS_CLIENT)
if ssl_minimum_version is not None:
context.minimum_version = ssl_minimum_version
else: # pyOpenSSL defaults to 'MINIMUM_SUPPORTED' so explicitly set TLSv1.2 here
context.minimum_version = TLSVersion.TLSv1_2
if ssl_maximum_version is not None:
context.maximum_version = ssl_maximum_version
# Unless we're given ciphers defer to either system ciphers in
# the case of OpenSSL 1.1.1+ or use our own secure default ciphers.
if ciphers:
context.set_ciphers(ciphers)
# Setting the default here, as we may have no ssl module on import
cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
if options is None:
options = 0
# SSLv2 is easily broken and is considered harmful and dangerous
options |= OP_NO_SSLv2
# SSLv3 has several problems and is now dangerous
options |= OP_NO_SSLv3
# Disable compression to prevent CRIME attacks for OpenSSL 1.0+
# (issue #309)
options |= OP_NO_COMPRESSION
# TLSv1.2 only. Unless set explicitly, do not request tickets.
# This may save some bandwidth on wire, and although the ticket is encrypted,
# there is a risk associated with it being on wire,
# if the server is not rotating its ticketing keys properly.
options |= OP_NO_TICKET
context.options |= options
if verify_flags is None:
verify_flags = 0
# In Python 3.13+ ssl.create_default_context() sets VERIFY_X509_PARTIAL_CHAIN
# and VERIFY_X509_STRICT so we do the same
if sys.version_info >= (3, 13):
verify_flags |= VERIFY_X509_PARTIAL_CHAIN
verify_flags |= VERIFY_X509_STRICT
context.verify_flags |= verify_flags
# Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is
# necessary for conditional client cert authentication with TLS 1.3.
# The attribute is None for OpenSSL <= 1.1.0 or does not exist when using
# an SSLContext created by pyOpenSSL.
if getattr(context, "post_handshake_auth", None) is not None:
context.post_handshake_auth = True
# The order of the below lines setting verify_mode and check_hostname
# matter due to safe-guards SSLContext has to prevent an SSLContext with
# check_hostname=True, verify_mode=NONE/OPTIONAL.
# We always set 'check_hostname=False' for pyOpenSSL so we rely on our own
# 'ssl.match_hostname()' implementation.
if cert_reqs == ssl.CERT_REQUIRED and not IS_PYOPENSSL:
context.verify_mode = cert_reqs
context.check_hostname = True
else:
context.check_hostname = False
context.verify_mode = cert_reqs
context.hostname_checks_common_name = False
if "SSLKEYLOGFILE" in os.environ:
sslkeylogfile = os.path.expandvars(os.environ.get("SSLKEYLOGFILE"))
else:
sslkeylogfile = None
if sslkeylogfile:
context.keylog_filename = sslkeylogfile
return context
@typing.overload
def ssl_wrap_socket(
sock: socket.socket,
keyfile: str | None = ...,
certfile: str | None = ...,
cert_reqs: int | None = ...,
ca_certs: str | None = ...,
server_hostname: str | None = ...,
ssl_version: int | None = ...,
ciphers: str | None = ...,
ssl_context: ssl.SSLContext | None = ...,
ca_cert_dir: str | None = ...,
key_password: str | None = ...,
ca_cert_data: None | str | bytes = ...,
tls_in_tls: typing.Literal[False] = ...,
) -> ssl.SSLSocket: ...
@typing.overload
def ssl_wrap_socket(
sock: socket.socket,
keyfile: str | None = ...,
certfile: str | None = ...,
cert_reqs: int | None = ...,
ca_certs: str | None = ...,
server_hostname: str | None = ...,
ssl_version: int | None = ...,
ciphers: str | None = ...,
ssl_context: ssl.SSLContext | None = ...,
ca_cert_dir: str | None = ...,
key_password: str | None = ...,
ca_cert_data: None | str | bytes = ...,
tls_in_tls: bool = ...,
) -> ssl.SSLSocket | SSLTransportType: ...
def ssl_wrap_socket(
sock: socket.socket,
keyfile: str | None = None,
certfile: str | None = None,
cert_reqs: int | None = None,
ca_certs: str | None = None,
server_hostname: str | None = None,
ssl_version: int | None = None,
ciphers: str | None = None,
ssl_context: ssl.SSLContext | None = None,
ca_cert_dir: str | None = None,
key_password: str | None = None,
ca_cert_data: None | str | bytes = None,
tls_in_tls: bool = False,
) -> ssl.SSLSocket | SSLTransportType:
"""
All arguments except for server_hostname, ssl_context, tls_in_tls, ca_cert_data and
ca_cert_dir have the same meaning as they do when using
:func:`ssl.create_default_context`, :meth:`ssl.SSLContext.load_cert_chain`,
:meth:`ssl.SSLContext.set_ciphers` and :meth:`ssl.SSLContext.wrap_socket`.
:param server_hostname:
When SNI is supported, the expected hostname of the certificate
:param ssl_context:
A pre-made :class:`SSLContext` object. If none is provided, one will
be created using :func:`create_urllib3_context`.
:param ciphers:
A string of ciphers we wish the client to support.
:param ca_cert_dir:
A directory containing CA certificates in multiple separate files, as
supported by OpenSSL's -CApath flag or the capath argument to
SSLContext.load_verify_locations().
:param key_password:
Optional password if the keyfile is encrypted.
:param ca_cert_data:
Optional string containing CA certificates in PEM format suitable for
passing as the cadata parameter to SSLContext.load_verify_locations()
:param tls_in_tls:
Use SSLTransport to wrap the existing socket.
"""
context = ssl_context
if context is None:
# Note: This branch of code and all the variables in it are only used in tests.
# We should consider deprecating and removing this code.
context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers)
if ca_certs or ca_cert_dir or ca_cert_data:
try:
context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data)
except OSError as e:
raise SSLError(e) from e
elif ssl_context is None and hasattr(context, "load_default_certs"):
# try to load OS default certs; works well on Windows.
context.load_default_certs()
# Attempt to detect if we get the goofy behavior of the
# keyfile being encrypted and OpenSSL asking for the
# passphrase via the terminal and instead error out.
if keyfile and key_password is None and _is_key_file_encrypted(keyfile):
raise SSLError("Client private key is encrypted, password is required")
if certfile:
if key_password is None:
context.load_cert_chain(certfile, keyfile)
else:
context.load_cert_chain(certfile, keyfile, key_password)
context.set_alpn_protocols(ALPN_PROTOCOLS)
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
return ssl_sock
def is_ipaddress(hostname: str | bytes) -> bool:
"""Detects whether the hostname given is an IPv4 or IPv6 address.
Also detects IPv6 addresses with Zone IDs.
:param str hostname: Hostname to examine.
:return: True if the hostname is an IP address, False otherwise.
"""
if isinstance(hostname, bytes):
# IDN A-label bytes are ASCII compatible.
hostname = hostname.decode("ascii")
return bool(_IPV4_RE.match(hostname) or _BRACELESS_IPV6_ADDRZ_RE.match(hostname))
def _is_key_file_encrypted(key_file: str) -> bool:
"""Detects if a key file is encrypted or not."""
with open(key_file) as f:
for line in f:
# Look for Proc-Type: 4,ENCRYPTED
if "ENCRYPTED" in line:
return True
return False
def _ssl_wrap_socket_impl(
sock: socket.socket,
ssl_context: ssl.SSLContext,
tls_in_tls: bool,
server_hostname: str | None = None,
) -> ssl.SSLSocket | SSLTransportType:
if tls_in_tls:
if not SSLTransport:
# Import error, ssl is not available.
raise ProxySchemeUnsupported(
"TLS in TLS requires support for the 'ssl' module"
)
SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context)
return SSLTransport(sock, ssl_context, server_hostname)
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
@@ -0,0 +1,153 @@
"""The match_hostname() function from Python 3.5, essential when using SSL."""
# Note: This file is under the PSF license as the code comes from the python
# stdlib. http://docs.python.org/3/license.html
# It is modified to remove commonName support.
from __future__ import annotations
import ipaddress
import re
import typing
from ipaddress import IPv4Address, IPv6Address
if typing.TYPE_CHECKING:
from .ssl_ import _TYPE_PEER_CERT_RET_DICT
__version__ = "3.5.0.1"
class CertificateError(ValueError):
pass
def _dnsname_match(
dn: typing.Any, hostname: str, max_wildcards: int = 1
) -> typing.Match[str] | None | bool:
"""Matching according to RFC 6125, section 6.4.3
http://tools.ietf.org/html/rfc6125#section-6.4.3
"""
pats = []
if not dn:
return False
# Ported from python3-syntax:
# leftmost, *remainder = dn.split(r'.')
parts = dn.split(r".")
leftmost = parts[0]
remainder = parts[1:]
wildcards = leftmost.count("*")
if wildcards > max_wildcards:
# Issue #17980: avoid denials of service by refusing more
# than one wildcard per fragment. A survey of established
# policy among SSL implementations showed it to be a
# reasonable choice.
raise CertificateError(
"too many wildcards in certificate DNS name: " + repr(dn)
)
# speed up common case w/o wildcards
if not wildcards:
return bool(dn.lower() == hostname.lower())
# RFC 6125, section 6.4.3, subitem 1.
# The client SHOULD NOT attempt to match a presented identifier in which
# the wildcard character comprises a label other than the left-most label.
if leftmost == "*":
# When '*' is a fragment by itself, it matches a non-empty dotless
# fragment.
pats.append("[^.]+")
elif leftmost.startswith("xn--") or hostname.startswith("xn--"):
# RFC 6125, section 6.4.3, subitem 3.
# The client SHOULD NOT attempt to match a presented identifier
# where the wildcard character is embedded within an A-label or
# U-label of an internationalized domain name.
pats.append(re.escape(leftmost))
else:
# Otherwise, '*' matches any dotless string, e.g. www*
pats.append(re.escape(leftmost).replace(r"\*", "[^.]*"))
# add the remaining fragments, ignore any wildcards
for frag in remainder:
pats.append(re.escape(frag))
pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE)
return pat.match(hostname)
def _ipaddress_match(ipname: str, host_ip: IPv4Address | IPv6Address) -> bool:
"""Exact matching of IP addresses.
RFC 9110 section 4.3.5: "A reference identity of IP-ID contains the decoded
bytes of the IP address. An IP version 4 address is 4 octets, and an IP
version 6 address is 16 octets. [...] A reference identity of type IP-ID
matches if the address is identical to an iPAddress value of the
subjectAltName extension of the certificate."
"""
# OpenSSL may add a trailing newline to a subjectAltName's IP address
# Divergence from upstream: ipaddress can't handle byte str
ip = ipaddress.ip_address(ipname.rstrip())
return bool(ip.packed == host_ip.packed)
def match_hostname(
cert: _TYPE_PEER_CERT_RET_DICT | None,
hostname: str,
hostname_checks_common_name: bool = False,
) -> None:
"""Verify that *cert* (in decoded format as returned by
SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125
rules are followed, but IP addresses are not accepted for *hostname*.
CertificateError is raised on failure. On success, the function
returns nothing.
"""
if not cert:
raise ValueError(
"empty or no certificate, match_hostname needs a "
"SSL socket or SSL context with either "
"CERT_OPTIONAL or CERT_REQUIRED"
)
try:
host_ip = ipaddress.ip_address(hostname)
except ValueError:
# Not an IP address (common case)
host_ip = None
dnsnames = []
san: tuple[tuple[str, str], ...] = cert.get("subjectAltName", ())
key: str
value: str
for key, value in san:
if key == "DNS":
if host_ip is None and _dnsname_match(value, hostname):
return
dnsnames.append(value)
elif key == "IP Address":
if host_ip is not None and _ipaddress_match(value, host_ip):
return
dnsnames.append(value)
# We only check 'commonName' if it's enabled and we're not verifying
# an IP address. IP addresses aren't valid within 'commonName'.
if hostname_checks_common_name and host_ip is None and not dnsnames:
for sub in cert.get("subject", ()):
for key, value in sub:
if key == "commonName":
if _dnsname_match(value, hostname):
return
dnsnames.append(
value
) # Defensive: for older PyPy and OpenSSL versions
if len(dnsnames) > 1:
raise CertificateError(
"hostname %r "
"doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames)))
)
elif len(dnsnames) == 1:
raise CertificateError(f"hostname {hostname!r} doesn't match {dnsnames[0]!r}")
else:
raise CertificateError("no appropriate subjectAltName fields were found")
@@ -0,0 +1,271 @@
from __future__ import annotations
import io
import socket
import ssl
import typing
from ..exceptions import ProxySchemeUnsupported
if typing.TYPE_CHECKING:
from typing_extensions import Self
from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT
_WriteBuffer = typing.Union[bytearray, memoryview]
_ReturnValue = typing.TypeVar("_ReturnValue")
SSL_BLOCKSIZE = 16384
class SSLTransport:
"""
The SSLTransport wraps an existing socket and establishes an SSL connection.
Contrary to Python's implementation of SSLSocket, it allows you to chain
multiple TLS connections together. It's particularly useful if you need to
implement TLS within TLS.
The class supports most of the socket API operations.
"""
@staticmethod
def _validate_ssl_context_for_tls_in_tls(ssl_context: ssl.SSLContext) -> None:
"""
Raises a ProxySchemeUnsupported if the provided ssl_context can't be used
for TLS in TLS.
The only requirement is that the ssl_context provides the 'wrap_bio'
methods.
"""
if not hasattr(ssl_context, "wrap_bio"):
raise ProxySchemeUnsupported(
"TLS in TLS requires SSLContext.wrap_bio() which isn't "
"available on non-native SSLContext"
)
def __init__(
self,
socket: socket.socket,
ssl_context: ssl.SSLContext,
server_hostname: str | None = None,
suppress_ragged_eofs: bool = True,
) -> None:
"""
Create an SSLTransport around socket using the provided ssl_context.
"""
self.incoming = ssl.MemoryBIO()
self.outgoing = ssl.MemoryBIO()
self.suppress_ragged_eofs = suppress_ragged_eofs
self.socket = socket
self.sslobj = ssl_context.wrap_bio(
self.incoming, self.outgoing, server_hostname=server_hostname
)
# Perform initial handshake.
self._ssl_io_loop(self.sslobj.do_handshake)
def __enter__(self) -> Self:
return self
def __exit__(self, *_: typing.Any) -> None:
self.close()
def fileno(self) -> int:
return self.socket.fileno()
def read(self, len: int = 1024, buffer: typing.Any | None = None) -> int | bytes:
return self._wrap_ssl_read(len, buffer)
def recv(self, buflen: int = 1024, flags: int = 0) -> int | bytes:
if flags != 0:
raise ValueError("non-zero flags not allowed in calls to recv")
return self._wrap_ssl_read(buflen)
def recv_into(
self,
buffer: _WriteBuffer,
nbytes: int | None = None,
flags: int = 0,
) -> None | int | bytes:
if flags != 0:
raise ValueError("non-zero flags not allowed in calls to recv_into")
if nbytes is None:
nbytes = len(buffer)
return self.read(nbytes, buffer)
def sendall(self, data: bytes, flags: int = 0) -> None:
if flags != 0:
raise ValueError("non-zero flags not allowed in calls to sendall")
count = 0
with memoryview(data) as view, view.cast("B") as byte_view:
amount = len(byte_view)
while count < amount:
v = self.send(byte_view[count:])
count += v
def send(self, data: bytes, flags: int = 0) -> int:
if flags != 0:
raise ValueError("non-zero flags not allowed in calls to send")
return self._ssl_io_loop(self.sslobj.write, data)
def makefile(
self,
mode: str,
buffering: int | None = None,
*,
encoding: str | None = None,
errors: str | None = None,
newline: str | None = None,
) -> typing.BinaryIO | typing.TextIO | socket.SocketIO:
"""
Python's httpclient uses makefile and buffered io when reading HTTP
messages and we need to support it.
This is unfortunately a copy and paste of socket.py makefile with small
changes to point to the socket directly.
"""
if not set(mode) <= {"r", "w", "b"}:
raise ValueError(f"invalid mode {mode!r} (only r, w, b allowed)")
writing = "w" in mode
reading = "r" in mode or not writing
assert reading or writing
binary = "b" in mode
rawmode = ""
if reading:
rawmode += "r"
if writing:
rawmode += "w"
raw = socket.SocketIO(self, rawmode) # type: ignore[arg-type]
self.socket._io_refs += 1 # type: ignore[attr-defined]
if buffering is None:
buffering = -1
if buffering < 0:
buffering = io.DEFAULT_BUFFER_SIZE
if buffering == 0:
if not binary:
raise ValueError("unbuffered streams must be binary")
return raw
buffer: typing.BinaryIO
if reading and writing:
buffer = io.BufferedRWPair(raw, raw, buffering) # type: ignore[assignment]
elif reading:
buffer = io.BufferedReader(raw, buffering)
else:
assert writing
buffer = io.BufferedWriter(raw, buffering)
if binary:
return buffer
text = io.TextIOWrapper(buffer, encoding, errors, newline)
text.mode = mode # type: ignore[misc]
return text
def unwrap(self) -> None:
self._ssl_io_loop(self.sslobj.unwrap)
def close(self) -> None:
self.socket.close()
@typing.overload
def getpeercert(
self, binary_form: typing.Literal[False] = ...
) -> _TYPE_PEER_CERT_RET_DICT | None: ...
@typing.overload
def getpeercert(self, binary_form: typing.Literal[True]) -> bytes | None: ...
def getpeercert(self, binary_form: bool = False) -> _TYPE_PEER_CERT_RET:
return self.sslobj.getpeercert(binary_form) # type: ignore[return-value]
def version(self) -> str | None:
return self.sslobj.version()
def cipher(self) -> tuple[str, str, int] | None:
return self.sslobj.cipher()
def selected_alpn_protocol(self) -> str | None:
return self.sslobj.selected_alpn_protocol()
def shared_ciphers(self) -> list[tuple[str, str, int]] | None:
return self.sslobj.shared_ciphers()
def compression(self) -> str | None:
return self.sslobj.compression()
def settimeout(self, value: float | None) -> None:
self.socket.settimeout(value)
def gettimeout(self) -> float | None:
return self.socket.gettimeout()
def _decref_socketios(self) -> None:
self.socket._decref_socketios() # type: ignore[attr-defined]
def _wrap_ssl_read(self, len: int, buffer: bytearray | None = None) -> int | bytes:
try:
return self._ssl_io_loop(self.sslobj.read, len, buffer)
except ssl.SSLError as e:
if e.errno == ssl.SSL_ERROR_EOF and self.suppress_ragged_eofs:
return 0 # eof, return 0.
else:
raise
# func is sslobj.do_handshake or sslobj.unwrap
@typing.overload
def _ssl_io_loop(self, func: typing.Callable[[], None]) -> None: ...
# func is sslobj.write, arg1 is data
@typing.overload
def _ssl_io_loop(self, func: typing.Callable[[bytes], int], arg1: bytes) -> int: ...
# func is sslobj.read, arg1 is len, arg2 is buffer
@typing.overload
def _ssl_io_loop(
self,
func: typing.Callable[[int, bytearray | None], bytes],
arg1: int,
arg2: bytearray | None,
) -> bytes: ...
def _ssl_io_loop(
self,
func: typing.Callable[..., _ReturnValue],
arg1: None | bytes | int = None,
arg2: bytearray | None = None,
) -> _ReturnValue:
"""Performs an I/O loop between incoming/outgoing and the socket."""
should_loop = True
ret = None
while should_loop:
errno = None
try:
if arg1 is None and arg2 is None:
ret = func()
elif arg2 is None:
ret = func(arg1)
else:
ret = func(arg1, arg2)
except ssl.SSLError as e:
if e.errno not in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE):
# WANT_READ, and WANT_WRITE are expected, others are not.
raise e
errno = e.errno
buf = self.outgoing.read()
self.socket.sendall(buf)
if errno is None:
should_loop = False
elif errno == ssl.SSL_ERROR_WANT_READ:
buf = self.socket.recv(SSL_BLOCKSIZE)
if buf:
self.incoming.write(buf)
else:
self.incoming.write_eof()
return typing.cast(_ReturnValue, ret)
@@ -0,0 +1,275 @@
from __future__ import annotations
import time
import typing
from enum import Enum
from socket import getdefaulttimeout
from ..exceptions import TimeoutStateError
if typing.TYPE_CHECKING:
from typing import Final
class _TYPE_DEFAULT(Enum):
# This value should never be passed to socket.settimeout() so for safety we use a -1.
# socket.settimout() raises a ValueError for negative values.
token = -1
_DEFAULT_TIMEOUT: Final[_TYPE_DEFAULT] = _TYPE_DEFAULT.token
_TYPE_TIMEOUT = typing.Optional[typing.Union[float, _TYPE_DEFAULT]]
class Timeout:
"""Timeout configuration.
Timeouts can be defined as a default for a pool:
.. code-block:: python
import urllib3
timeout = urllib3.util.Timeout(connect=2.0, read=7.0)
http = urllib3.PoolManager(timeout=timeout)
resp = http.request("GET", "https://example.com/")
print(resp.status)
Or per-request (which overrides the default for the pool):
.. code-block:: python
response = http.request("GET", "https://example.com/", timeout=Timeout(10))
Timeouts can be disabled by setting all the parameters to ``None``:
.. code-block:: python
no_timeout = Timeout(connect=None, read=None)
response = http.request("GET", "https://example.com/", timeout=no_timeout)
:param total:
This combines the connect and read timeouts into one; the read timeout
will be set to the time leftover from the connect attempt. In the
event that both a connect timeout and a total are specified, or a read
timeout and a total are specified, the shorter timeout will be applied.
Defaults to None.
:type total: int, float, or None
:param connect:
The maximum amount of time (in seconds) to wait for a connection
attempt to a server to succeed. Omitting the parameter will default the
connect timeout to the system default, probably `the global default
timeout in socket.py
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
None will set an infinite timeout for connection attempts.
:type connect: int, float, or None
:param read:
The maximum amount of time (in seconds) to wait between consecutive
read operations for a response from the server. Omitting the parameter
will default the read timeout to the system default, probably `the
global default timeout in socket.py
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
None will set an infinite timeout.
:type read: int, float, or None
.. note::
Many factors can affect the total amount of time for urllib3 to return
an HTTP response.
For example, Python's DNS resolver does not obey the timeout specified
on the socket. Other factors that can affect total request time include
high CPU load, high swap, the program running at a low priority level,
or other behaviors.
In addition, the read and total timeouts only measure the time between
read operations on the socket connecting the client and the server,
not the total amount of time for the request to return a complete
response. For most requests, the timeout is raised because the server
has not sent the first byte in the specified time. This is not always
the case; if a server streams one byte every fifteen seconds, a timeout
of 20 seconds will not trigger, even though the request will take
several minutes to complete.
"""
#: A sentinel object representing the default timeout value
DEFAULT_TIMEOUT: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT
def __init__(
self,
total: _TYPE_TIMEOUT = None,
connect: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
read: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
) -> None:
self._connect = self._validate_timeout(connect, "connect")
self._read = self._validate_timeout(read, "read")
self.total = self._validate_timeout(total, "total")
self._start_connect: float | None = None
def __repr__(self) -> str:
return f"{type(self).__name__}(connect={self._connect!r}, read={self._read!r}, total={self.total!r})"
# __str__ provided for backwards compatibility
__str__ = __repr__
@staticmethod
def resolve_default_timeout(timeout: _TYPE_TIMEOUT) -> float | None:
return getdefaulttimeout() if timeout is _DEFAULT_TIMEOUT else timeout
@classmethod
def _validate_timeout(cls, value: _TYPE_TIMEOUT, name: str) -> _TYPE_TIMEOUT:
"""Check that a timeout attribute is valid.
:param value: The timeout value to validate
:param name: The name of the timeout attribute to validate. This is
used to specify in error messages.
:return: The validated and casted version of the given value.
:raises ValueError: If it is a numeric value less than or equal to
zero, or the type is not an integer, float, or None.
"""
if value is None or value is _DEFAULT_TIMEOUT:
return value
if isinstance(value, bool):
raise ValueError(
"Timeout cannot be a boolean value. It must "
"be an int, float or None."
)
try:
float(value)
except (TypeError, ValueError):
raise ValueError(
"Timeout value %s was %s, but it must be an "
"int, float or None." % (name, value)
) from None
try:
if value <= 0:
raise ValueError(
"Attempted to set %s timeout to %s, but the "
"timeout cannot be set to a value less "
"than or equal to 0." % (name, value)
)
except TypeError:
raise ValueError(
"Timeout value %s was %s, but it must be an "
"int, float or None." % (name, value)
) from None
return value
@classmethod
def from_float(cls, timeout: _TYPE_TIMEOUT) -> Timeout:
"""Create a new Timeout from a legacy timeout value.
The timeout value used by httplib.py sets the same timeout on the
connect(), and recv() socket requests. This creates a :class:`Timeout`
object that sets the individual timeouts to the ``timeout`` value
passed to this function.
:param timeout: The legacy timeout value.
:type timeout: integer, float, :attr:`urllib3.util.Timeout.DEFAULT_TIMEOUT`, or None
:return: Timeout object
:rtype: :class:`Timeout`
"""
return Timeout(read=timeout, connect=timeout)
def clone(self) -> Timeout:
"""Create a copy of the timeout object
Timeout properties are stored per-pool but each request needs a fresh
Timeout object to ensure each one has its own start/stop configured.
:return: a copy of the timeout object
:rtype: :class:`Timeout`
"""
# We can't use copy.deepcopy because that will also create a new object
# for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to
# detect the user default.
return Timeout(connect=self._connect, read=self._read, total=self.total)
def start_connect(self) -> float:
"""Start the timeout clock, used during a connect() attempt
:raises urllib3.exceptions.TimeoutStateError: if you attempt
to start a timer that has been started already.
"""
if self._start_connect is not None:
raise TimeoutStateError("Timeout timer has already been started.")
self._start_connect = time.monotonic()
return self._start_connect
def get_connect_duration(self) -> float:
"""Gets the time elapsed since the call to :meth:`start_connect`.
:return: Elapsed time in seconds.
:rtype: float
:raises urllib3.exceptions.TimeoutStateError: if you attempt
to get duration for a timer that hasn't been started.
"""
if self._start_connect is None:
raise TimeoutStateError(
"Can't get connect duration for timer that has not started."
)
return time.monotonic() - self._start_connect
@property
def connect_timeout(self) -> _TYPE_TIMEOUT:
"""Get the value to use when setting a connection timeout.
This will be a positive float or integer, the value None
(never timeout), or the default system timeout.
:return: Connect timeout.
:rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
"""
if self.total is None:
return self._connect
if self._connect is None or self._connect is _DEFAULT_TIMEOUT:
return self.total
return min(self._connect, self.total) # type: ignore[type-var]
@property
def read_timeout(self) -> float | None:
"""Get the value for the read timeout.
This assumes some time has elapsed in the connection timeout and
computes the read timeout appropriately.
If self.total is set, the read timeout is dependent on the amount of
time taken by the connect timeout. If the connection time has not been
established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be
raised.
:return: Value to use for the read timeout.
:rtype: int, float or None
:raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
has not yet been called on this object.
"""
if (
self.total is not None
and self.total is not _DEFAULT_TIMEOUT
and self._read is not None
and self._read is not _DEFAULT_TIMEOUT
):
# In case the connect timeout has not yet been established.
if self._start_connect is None:
return self._read
return max(0, min(self.total - self.get_connect_duration(), self._read))
elif self.total is not None and self.total is not _DEFAULT_TIMEOUT:
return max(0, self.total - self.get_connect_duration())
else:
return self.resolve_default_timeout(self._read)
@@ -0,0 +1,469 @@
from __future__ import annotations
import re
import typing
from ..exceptions import LocationParseError
from .util import to_str
# We only want to normalize urls with an HTTP(S) scheme.
# urllib3 infers URLs without a scheme (None) to be http.
_NORMALIZABLE_SCHEMES = ("http", "https", None)
# Almost all of these patterns were derived from the
# 'rfc3986' module: https://github.com/python-hyper/rfc3986
_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}")
_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)")
_URI_RE = re.compile(
r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?"
r"(?://([^\\/?#]*))?"
r"([^?#]*)"
r"(?:\?([^#]*))?"
r"(?:#(.*))?$",
re.UNICODE | re.DOTALL,
)
_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"
_HEX_PAT = "[0-9A-Fa-f]{1,4}"
_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT)
_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT}
_variations = [
# 6( h16 ":" ) ls32
"(?:%(hex)s:){6}%(ls32)s",
# "::" 5( h16 ":" ) ls32
"::(?:%(hex)s:){5}%(ls32)s",
# [ h16 ] "::" 4( h16 ":" ) ls32
"(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",
# [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
"(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",
# [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
"(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",
# [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
"(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",
# [ *4( h16 ":" ) h16 ] "::" ls32
"(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",
# [ *5( h16 ":" ) h16 ] "::" h16
"(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",
# [ *6( h16 ":" ) h16 ] "::"
"(?:(?:%(hex)s:){0,6}%(hex)s)?::",
]
_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~"
_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"
_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"
_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]"
_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*"
_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$")
_IPV4_RE = re.compile("^" + _IPV4_PAT + "$")
_IPV6_RE = re.compile("^" + _IPV6_PAT + "$")
_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$")
_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$")
_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$")
_HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % (
_REG_NAME_PAT,
_IPV4_PAT,
_IPV6_ADDRZ_PAT,
)
_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL)
_UNRESERVED_CHARS = set(
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~"
)
_SUB_DELIM_CHARS = set("!$&'()*+,;=")
_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"}
_PATH_CHARS = _USERINFO_CHARS | {"@", "/"}
_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"}
class Url(
typing.NamedTuple(
"Url",
[
("scheme", typing.Optional[str]),
("auth", typing.Optional[str]),
("host", typing.Optional[str]),
("port", typing.Optional[int]),
("path", typing.Optional[str]),
("query", typing.Optional[str]),
("fragment", typing.Optional[str]),
],
)
):
"""
Data structure for representing an HTTP URL. Used as a return value for
:func:`parse_url`. Both the scheme and host are normalized as they are
both case-insensitive according to RFC 3986.
"""
def __new__( # type: ignore[no-untyped-def]
cls,
scheme: str | None = None,
auth: str | None = None,
host: str | None = None,
port: int | None = None,
path: str | None = None,
query: str | None = None,
fragment: str | None = None,
):
if path and not path.startswith("/"):
path = "/" + path
if scheme is not None:
scheme = scheme.lower()
return super().__new__(cls, scheme, auth, host, port, path, query, fragment)
@property
def hostname(self) -> str | None:
"""For backwards-compatibility with urlparse. We're nice like that."""
return self.host
@property
def request_uri(self) -> str:
"""Absolute path including the query string."""
uri = self.path or "/"
if self.query is not None:
uri += "?" + self.query
return uri
@property
def authority(self) -> str | None:
"""
Authority component as defined in RFC 3986 3.2.
This includes userinfo (auth), host and port.
i.e.
userinfo@host:port
"""
userinfo = self.auth
netloc = self.netloc
if netloc is None or userinfo is None:
return netloc
else:
return f"{userinfo}@{netloc}"
@property
def netloc(self) -> str | None:
"""
Network location including host and port.
If you need the equivalent of urllib.parse's ``netloc``,
use the ``authority`` property instead.
"""
if self.host is None:
return None
if self.port:
return f"{self.host}:{self.port}"
return self.host
@property
def url(self) -> str:
"""
Convert self into a url
This function should more or less round-trip with :func:`.parse_url`. The
returned url may not be exactly the same as the url inputted to
:func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls
with a blank port will have : removed).
Example:
.. code-block:: python
import urllib3
U = urllib3.util.parse_url("https://google.com/mail/")
print(U.url)
# "https://google.com/mail/"
print( urllib3.util.Url("https", "username:password",
"host.com", 80, "/path", "query", "fragment"
).url
)
# "https://username:[email protected]:80/path?query#fragment"
"""
scheme, auth, host, port, path, query, fragment = self
url = ""
# We use "is not None" we want things to happen with empty strings (or 0 port)
if scheme is not None:
url += scheme + "://"
if auth is not None:
url += auth + "@"
if host is not None:
url += host
if port is not None:
url += ":" + str(port)
if path is not None:
url += path
if query is not None:
url += "?" + query
if fragment is not None:
url += "#" + fragment
return url
def __str__(self) -> str:
return self.url
@typing.overload
def _encode_invalid_chars(
component: str, allowed_chars: typing.Container[str]
) -> str: # Abstract
...
@typing.overload
def _encode_invalid_chars(
component: None, allowed_chars: typing.Container[str]
) -> None: # Abstract
...
def _encode_invalid_chars(
component: str | None, allowed_chars: typing.Container[str]
) -> str | None:
"""Percent-encodes a URI component without reapplying
onto an already percent-encoded component.
"""
if component is None:
return component
component = to_str(component)
# Normalize existing percent-encoded bytes.
# Try to see if the component we're encoding is already percent-encoded
# so we can skip all '%' characters but still encode all others.
component, percent_encodings = _PERCENT_RE.subn(
lambda match: match.group(0).upper(), component
)
uri_bytes = component.encode("utf-8", "surrogatepass")
is_percent_encoded = percent_encodings == uri_bytes.count(b"%")
encoded_component = bytearray()
for i in range(0, len(uri_bytes)):
# Will return a single character bytestring
byte = uri_bytes[i : i + 1]
byte_ord = ord(byte)
if (is_percent_encoded and byte == b"%") or (
byte_ord < 128 and byte.decode() in allowed_chars
):
encoded_component += byte
continue
encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper()))
return encoded_component.decode()
def _remove_path_dot_segments(path: str) -> str:
# See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code
segments = path.split("/") # Turn the path into a list of segments
output = [] # Initialize the variable to use to store output
for segment in segments:
# '.' is the current directory, so ignore it, it is superfluous
if segment == ".":
continue
# Anything other than '..', should be appended to the output
if segment != "..":
output.append(segment)
# In this case segment == '..', if we can, we should pop the last
# element
elif output:
output.pop()
# If the path starts with '/' and the output is empty or the first string
# is non-empty
if path.startswith("/") and (not output or output[0]):
output.insert(0, "")
# If the path starts with '/.' or '/..' ensure we add one more empty
# string to add a trailing '/'
if path.endswith(("/.", "/..")):
output.append("")
return "/".join(output)
@typing.overload
def _normalize_host(host: None, scheme: str | None) -> None: ...
@typing.overload
def _normalize_host(host: str, scheme: str | None) -> str: ...
def _normalize_host(host: str | None, scheme: str | None) -> str | None:
if host:
if scheme in _NORMALIZABLE_SCHEMES:
is_ipv6 = _IPV6_ADDRZ_RE.match(host)
if is_ipv6:
# IPv6 hosts of the form 'a::b%zone' are encoded in a URL as
# such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID
# separator as necessary to return a valid RFC 4007 scoped IP.
match = _ZONE_ID_RE.search(host)
if match:
start, end = match.span(1)
zone_id = host[start:end]
if zone_id.startswith("%25") and zone_id != "%25":
zone_id = zone_id[3:]
else:
zone_id = zone_id[1:]
zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS)
return f"{host[:start].lower()}%{zone_id}{host[end:]}"
else:
return host.lower()
elif not _IPV4_RE.match(host):
return to_str(
b".".join([_idna_encode(label) for label in host.split(".")]),
"ascii",
)
return host
def _idna_encode(name: str) -> bytes:
if not name.isascii():
try:
import idna
except ImportError:
raise LocationParseError(
"Unable to parse URL without the 'idna' module"
) from None
try:
return idna.encode(name.lower(), strict=True, std3_rules=True)
except idna.IDNAError:
raise LocationParseError(
f"Name '{name}' is not a valid IDNA label"
) from None
return name.lower().encode("ascii")
def _encode_target(target: str) -> str:
"""Percent-encodes a request target so that there are no invalid characters
Pre-condition for this function is that 'target' must start with '/'.
If that is the case then _TARGET_RE will always produce a match.
"""
match = _TARGET_RE.match(target)
if not match: # Defensive:
raise LocationParseError(f"{target!r} is not a valid request URI")
path, query = match.groups()
encoded_target = _encode_invalid_chars(path, _PATH_CHARS)
if query is not None:
query = _encode_invalid_chars(query, _QUERY_CHARS)
encoded_target += "?" + query
return encoded_target
def parse_url(url: str) -> Url:
"""
Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
performed to parse incomplete urls. Fields not provided will be None.
This parser is RFC 3986 and RFC 6874 compliant.
The parser logic and helper functions are based heavily on
work done in the ``rfc3986`` module.
:param str url: URL to parse into a :class:`.Url` namedtuple.
Partly backwards-compatible with :mod:`urllib.parse`.
Example:
.. code-block:: python
import urllib3
print( urllib3.util.parse_url('http://google.com/mail/'))
# Url(scheme='http', host='google.com', port=None, path='/mail/', ...)
print( urllib3.util.parse_url('google.com:80'))
# Url(scheme=None, host='google.com', port=80, path=None, ...)
print( urllib3.util.parse_url('/foo?bar'))
# Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
"""
if not url:
# Empty
return Url()
source_url = url
if not _SCHEME_RE.search(url):
url = "//" + url
scheme: str | None
authority: str | None
auth: str | None
host: str | None
port: str | None
port_int: int | None
path: str | None
query: str | None
fragment: str | None
try:
scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr]
normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES
if scheme:
scheme = scheme.lower()
if authority:
auth, _, host_port = authority.rpartition("@")
auth = auth or None
host, port = _HOST_PORT_RE.match(host_port).groups() # type: ignore[union-attr]
if auth and normalize_uri:
auth = _encode_invalid_chars(auth, _USERINFO_CHARS)
if port == "":
port = None
else:
auth, host, port = None, None, None
if port is not None:
port_int = int(port)
if not (0 <= port_int <= 65535):
raise LocationParseError(url)
else:
port_int = None
host = _normalize_host(host, scheme)
if normalize_uri and path:
path = _remove_path_dot_segments(path)
path = _encode_invalid_chars(path, _PATH_CHARS)
if normalize_uri and query:
query = _encode_invalid_chars(query, _QUERY_CHARS)
if normalize_uri and fragment:
fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS)
except (ValueError, AttributeError) as e:
raise LocationParseError(source_url) from e
# For the sake of backwards compatibility we put empty
# string values for path if there are any defined values
# beyond the path in the URL.
# TODO: Remove this when we break backwards compatibility.
if not path:
if query is not None or fragment is not None:
path = ""
else:
path = None
return Url(
scheme=scheme,
auth=auth,
host=host,
port=port_int,
path=path,
query=query,
fragment=fragment,
)
@@ -0,0 +1,42 @@
from __future__ import annotations
import typing
from types import TracebackType
def to_bytes(
x: str | bytes, encoding: str | None = None, errors: str | None = None
) -> bytes:
if isinstance(x, bytes):
return x
elif not isinstance(x, str):
raise TypeError(f"not expecting type {type(x).__name__}")
if encoding or errors:
return x.encode(encoding or "utf-8", errors=errors or "strict")
return x.encode()
def to_str(
x: str | bytes, encoding: str | None = None, errors: str | None = None
) -> str:
if isinstance(x, str):
return x
elif not isinstance(x, bytes):
raise TypeError(f"not expecting type {type(x).__name__}")
if encoding or errors:
return x.decode(encoding or "utf-8", errors=errors or "strict")
return x.decode()
def reraise(
tp: type[BaseException] | None,
value: BaseException,
tb: TracebackType | None = None,
) -> typing.NoReturn:
try:
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
finally:
value = None # type: ignore[assignment]
tb = None
@@ -0,0 +1,124 @@
from __future__ import annotations
import select
import socket
from functools import partial
__all__ = ["wait_for_read", "wait_for_write"]
# How should we wait on sockets?
#
# There are two types of APIs you can use for waiting on sockets: the fancy
# modern stateful APIs like epoll/kqueue, and the older stateless APIs like
# select/poll. The stateful APIs are more efficient when you have a lots of
# sockets to keep track of, because you can set them up once and then use them
# lots of times. But we only ever want to wait on a single socket at a time
# and don't want to keep track of state, so the stateless APIs are actually
# more efficient. So we want to use select() or poll().
#
# Now, how do we choose between select() and poll()? On traditional Unixes,
# select() has a strange calling convention that makes it slow, or fail
# altogether, for high-numbered file descriptors. The point of poll() is to fix
# that, so on Unixes, we prefer poll().
#
# On Windows, there is no poll() (or at least Python doesn't provide a wrapper
# for it), but that's OK, because on Windows, select() doesn't have this
# strange calling convention; plain select() works fine.
#
# So: on Windows we use select(), and everywhere else we use poll(). We also
# fall back to select() in case poll() is somehow broken or missing.
def select_wait_for_socket(
sock: socket.socket,
read: bool = False,
write: bool = False,
timeout: float | None = None,
) -> bool:
if not read and not write:
raise RuntimeError("must specify at least one of read=True, write=True")
rcheck = []
wcheck = []
if read:
rcheck.append(sock)
if write:
wcheck.append(sock)
# When doing a non-blocking connect, most systems signal success by
# marking the socket writable. Windows, though, signals success by marked
# it as "exceptional". We paper over the difference by checking the write
# sockets for both conditions. (The stdlib selectors module does the same
# thing.)
fn = partial(select.select, rcheck, wcheck, wcheck)
rready, wready, xready = fn(timeout)
return bool(rready or wready or xready)
def poll_wait_for_socket(
sock: socket.socket,
read: bool = False,
write: bool = False,
timeout: float | None = None,
) -> bool:
if not read and not write:
raise RuntimeError("must specify at least one of read=True, write=True")
mask = 0
if read:
mask |= select.POLLIN
if write:
mask |= select.POLLOUT
poll_obj = select.poll()
poll_obj.register(sock, mask)
# For some reason, poll() takes timeout in milliseconds
def do_poll(t: float | None) -> list[tuple[int, int]]:
if t is not None:
t *= 1000
return poll_obj.poll(t)
return bool(do_poll(timeout))
def _have_working_poll() -> bool:
# Apparently some systems have a select.poll that fails as soon as you try
# to use it, either due to strange configuration or broken monkeypatching
# from libraries like eventlet/greenlet.
try:
poll_obj = select.poll()
poll_obj.poll(0)
except (AttributeError, OSError):
return False
else:
return True
def wait_for_socket(
sock: socket.socket,
read: bool = False,
write: bool = False,
timeout: float | None = None,
) -> bool:
# We delay choosing which implementation to use until the first time we're
# called. We could do it at import time, but then we might make the wrong
# decision if someone goes wild with monkeypatching select.poll after
# we're imported.
global wait_for_socket
if _have_working_poll():
wait_for_socket = poll_wait_for_socket
elif hasattr(select, "select"):
wait_for_socket = select_wait_for_socket
return wait_for_socket(sock, read, write, timeout)
def wait_for_read(sock: socket.socket, timeout: float | None = None) -> bool:
"""Waits for reading to be available on a given socket.
Returns True if the socket is readable, or False if the timeout expired.
"""
return wait_for_socket(sock, read=True, timeout=timeout)
def wait_for_write(sock: socket.socket, timeout: float | None = None) -> bool:
"""Waits for writing to be available on a given socket.
Returns True if the socket is readable, or False if the timeout expired.
"""
return wait_for_socket(sock, write=True, timeout=timeout)
@@ -0,0 +1,56 @@
# ruff: noqa: F401
from ._api import YouTubeTranscriptApi
from ._transcripts import (
TranscriptList,
Transcript,
FetchedTranscript,
FetchedTranscriptSnippet,
)
from ._errors import (
YouTubeTranscriptApiException,
CookieError,
CookiePathInvalid,
CookieInvalid,
TranscriptsDisabled,
NoTranscriptFound,
CouldNotRetrieveTranscript,
VideoUnavailable,
VideoUnplayable,
IpBlocked,
RequestBlocked,
NotTranslatable,
TranslationLanguageNotAvailable,
FailedToCreateConsentCookie,
YouTubeRequestFailed,
InvalidVideoId,
AgeRestricted,
YouTubeDataUnparsable,
PoTokenRequired,
)
__all__ = [
"YouTubeTranscriptApi",
"TranscriptList",
"Transcript",
"FetchedTranscript",
"FetchedTranscriptSnippet",
"YouTubeTranscriptApiException",
"CookieError",
"CookiePathInvalid",
"CookieInvalid",
"TranscriptsDisabled",
"NoTranscriptFound",
"CouldNotRetrieveTranscript",
"VideoUnavailable",
"VideoUnplayable",
"IpBlocked",
"RequestBlocked",
"NotTranslatable",
"TranslationLanguageNotAvailable",
"FailedToCreateConsentCookie",
"YouTubeRequestFailed",
"InvalidVideoId",
"AgeRestricted",
"YouTubeDataUnparsable",
"PoTokenRequired",
]
@@ -0,0 +1,15 @@
import sys
import logging
from ._cli import YouTubeTranscriptCli
def main():
logging.basicConfig()
print(YouTubeTranscriptCli(sys.argv[1:]).run())
if __name__ == "__main__":
main()
@@ -0,0 +1,127 @@
from typing import Optional, Iterable
from requests import Session
from requests.adapters import HTTPAdapter
from urllib3 import Retry
from .proxies import ProxyConfig
from ._transcripts import TranscriptListFetcher, FetchedTranscript, TranscriptList
class YouTubeTranscriptApi:
def __init__(
self,
proxy_config: Optional[ProxyConfig] = None,
http_client: Optional[Session] = None,
):
"""
Note on thread-safety: As this class will initialize a `requests.Session`
object, it is not thread-safe. Make sure to initialize an instance of
`YouTubeTranscriptApi` per thread, if used in a multi-threading scenario!
:param proxy_config: an optional ProxyConfig object, defining proxies used for
all network requests. This can be used to work around your IP being blocked
by YouTube, as described in the "Working around IP bans" section of the
README
(https://github.com/jdepoix/youtube-transcript-api?tab=readme-ov-file#working-around-ip-bans-requestblocked-or-ipblocked-exception)
:param http_client: You can optionally pass in a requests.Session object, if you
manually want to share cookies between different instances of
`YouTubeTranscriptApi`, overwrite defaults, specify SSL certificates, etc.
"""
http_client = Session() if http_client is None else http_client
http_client.headers.update({"Accept-Language": "en-US"})
# Cookie auth has been temporarily disabled, as it is not working properly with
# YouTube's most recent changes.
# if cookie_path is not None:
# http_client.cookies = _load_cookie_jar(cookie_path)
if proxy_config is not None:
http_client.proxies = proxy_config.to_requests_dict()
if proxy_config.prevent_keeping_connections_alive:
http_client.headers.update({"Connection": "close"})
if proxy_config.retries_when_blocked > 0:
retry_config = Retry(
total=proxy_config.retries_when_blocked,
status_forcelist=[429],
)
http_client.mount("http://", HTTPAdapter(max_retries=retry_config))
http_client.mount("https://", HTTPAdapter(max_retries=retry_config))
self._fetcher = TranscriptListFetcher(http_client, proxy_config=proxy_config)
def fetch(
self,
video_id: str,
languages: Iterable[str] = ("en",),
preserve_formatting: bool = False,
) -> FetchedTranscript:
"""
Retrieves the transcript for a single video. This is just a shortcut for
calling:
`YouTubeTranscriptApi().list(video_id).find_transcript(languages).fetch(preserve_formatting=preserve_formatting)`
:param video_id: the ID of the video you want to retrieve the transcript for.
Make sure that this is the actual ID, NOT the full URL to the video!
:param languages: A list of language codes in a descending priority. For
example, if this is set to ["de", "en"] it will first try to fetch the
german transcript (de) and then fetch the english transcript (en) if
it fails to do so. This defaults to ["en"].
:param preserve_formatting: whether to keep select HTML text formatting
"""
return (
self.list(video_id)
.find_transcript(languages)
.fetch(preserve_formatting=preserve_formatting)
)
def list(
self,
video_id: str,
) -> TranscriptList:
"""
Retrieves the list of transcripts which are available for a given video. It
returns a `TranscriptList` object which is iterable and provides methods to
filter the list of transcripts for specific languages. While iterating over
the `TranscriptList` the individual transcripts are represented by
`Transcript` objects, which provide metadata and can either be fetched by
calling `transcript.fetch()` or translated by calling `transcript.translate(
'en')`. Example:
```
ytt_api = YouTubeTranscriptApi()
# retrieve the available transcripts
transcript_list = ytt_api.list('video_id')
# iterate over all available transcripts
for transcript in transcript_list:
# the Transcript object provides metadata properties
print(
transcript.video_id,
transcript.language,
transcript.language_code,
# whether it has been manually created or generated by YouTube
transcript.is_generated,
# a list of languages the transcript can be translated to
transcript.translation_languages,
)
# fetch the actual transcript data
print(transcript.fetch())
# translating the transcript will return another transcript object
print(transcript.translate('en').fetch())
# you can also directly filter for the language you are looking for, using the transcript list
transcript = transcript_list.find_transcript(['de', 'en'])
# or just filter for manually created transcripts
transcript = transcript_list.find_manually_created_transcript(['de', 'en'])
# or automatically generated ones
transcript = transcript_list.find_generated_transcript(['de', 'en'])
```
:param video_id: the ID of the video you want to retrieve the transcript for.
Make sure that this is the actual ID, NOT the full URL to the video!
"""
return self._fetcher.fetch(video_id)
@@ -0,0 +1,203 @@
import argparse
from importlib.metadata import PackageNotFoundError, version
from typing import List
from .proxies import GenericProxyConfig, WebshareProxyConfig
from .formatters import FormatterLoader
from ._api import YouTubeTranscriptApi, FetchedTranscript, TranscriptList
class YouTubeTranscriptCli:
def __init__(self, args: List[str]):
self._args = args
def run(self) -> str:
parsed_args = self._parse_args()
if parsed_args.exclude_manually_created and parsed_args.exclude_generated:
return ""
proxy_config = None
if parsed_args.http_proxy != "" or parsed_args.https_proxy != "":
proxy_config = GenericProxyConfig(
http_url=parsed_args.http_proxy,
https_url=parsed_args.https_proxy,
)
if (
parsed_args.webshare_proxy_username is not None
or parsed_args.webshare_proxy_password is not None
):
proxy_config = WebshareProxyConfig(
proxy_username=parsed_args.webshare_proxy_username,
proxy_password=parsed_args.webshare_proxy_password,
)
transcripts = []
exceptions = []
ytt_api = YouTubeTranscriptApi(
proxy_config=proxy_config,
)
for video_id in parsed_args.video_ids:
try:
transcript_list = ytt_api.list(video_id)
if parsed_args.list_transcripts:
transcripts.append(transcript_list)
else:
transcripts.append(
self._fetch_transcript(
parsed_args,
transcript_list,
)
)
except Exception as exception:
exceptions.append(exception)
print_sections = [str(exception) for exception in exceptions]
if transcripts:
if parsed_args.list_transcripts:
print_sections.extend(
str(transcript_list) for transcript_list in transcripts
)
else:
print_sections.append(
FormatterLoader()
.load(parsed_args.format)
.format_transcripts(transcripts)
)
return "\n\n".join(print_sections)
def _fetch_transcript(
self,
parsed_args,
transcript_list: TranscriptList,
) -> FetchedTranscript:
if parsed_args.exclude_manually_created:
transcript = transcript_list.find_generated_transcript(
parsed_args.languages
)
elif parsed_args.exclude_generated:
transcript = transcript_list.find_manually_created_transcript(
parsed_args.languages
)
else:
transcript = transcript_list.find_transcript(parsed_args.languages)
if parsed_args.translate:
transcript = transcript.translate(parsed_args.translate)
return transcript.fetch()
def _get_version(self):
try:
return version("youtube-transcript-api")
except PackageNotFoundError:
return "unknown"
def _parse_args(self):
parser = argparse.ArgumentParser(
description=(
"This is a python API which allows you to get the transcripts/subtitles for a given YouTube video. "
"It also works for automatically generated subtitles and it does not require a headless browser, like "
"other selenium based solutions do!"
)
)
parser.add_argument(
"--version",
action="version",
version=f"%(prog)s, version {self._get_version()}",
)
parser.add_argument(
"--list-transcripts",
action="store_const",
const=True,
default=False,
help="This will list the languages in which the given videos are available in.",
)
parser.add_argument(
"video_ids", nargs="+", type=str, help="List of YouTube video IDs."
)
parser.add_argument(
"--languages",
nargs="*",
default=[
"en",
],
type=str,
help=(
'A list of language codes in a descending priority. For example, if this is set to "de en" it will '
"first try to fetch the german transcript (de) and then fetch the english transcript (en) if it fails "
"to do so. As I can't provide a complete list of all working language codes with full certainty, you "
"may have to play around with the language codes a bit, to find the one which is working for you!"
),
)
parser.add_argument(
"--exclude-generated",
action="store_const",
const=True,
default=False,
help="If this flag is set transcripts which have been generated by YouTube will not be retrieved.",
)
parser.add_argument(
"--exclude-manually-created",
action="store_const",
const=True,
default=False,
help="If this flag is set transcripts which have been manually created will not be retrieved.",
)
parser.add_argument(
"--format",
type=str,
default="pretty",
choices=tuple(FormatterLoader.TYPES.keys()),
)
parser.add_argument(
"--translate",
default="",
help=(
"The language code for the language you want this transcript to be translated to. Use the "
"--list-transcripts feature to find out which languages are translatable and which translation "
"languages are available."
),
)
parser.add_argument(
"--webshare-proxy-username",
default=None,
type=str,
help='Specify your Webshare "Proxy Username" found at https://dashboard.webshare.io/proxy/settings',
)
parser.add_argument(
"--webshare-proxy-password",
default=None,
type=str,
help='Specify your Webshare "Proxy Password" found at https://dashboard.webshare.io/proxy/settings',
)
parser.add_argument(
"--http-proxy",
default="",
metavar="URL",
help="Use the specified HTTP proxy.",
)
parser.add_argument(
"--https-proxy",
default="",
metavar="URL",
help="Use the specified HTTPS proxy.",
)
# Cookie auth has been temporarily disabled, as it is not working properly with
# YouTube's most recent changes.
# parser.add_argument(
# "--cookies",
# default=None,
# help="The cookie file that will be used for authorization with youtube.",
# )
return self._sanitize_video_ids(parser.parse_args(self._args))
def _sanitize_video_ids(self, args):
args.video_ids = [video_id.replace("\\", "") for video_id in args.video_ids]
return args
@@ -0,0 +1,272 @@
from pathlib import Path
from typing import Iterable, Optional, List
from requests import HTTPError
from ._settings import WATCH_URL
from .proxies import ProxyConfig, GenericProxyConfig, WebshareProxyConfig
class YouTubeTranscriptApiException(Exception):
pass
class CookieError(YouTubeTranscriptApiException):
pass
class CookiePathInvalid(CookieError):
def __init__(
self, cookie_path: Path
): # pragma: no cover until cookie authentication is re-implemented
super().__init__(f"Can't load the provided cookie file: {cookie_path}")
class CookieInvalid(CookieError):
def __init__(
self, cookie_path: Path
): # pragma: no cover until cookie authentication is re-implemented
super().__init__(
f"The cookies provided are not valid (may have expired): {cookie_path}"
)
class CouldNotRetrieveTranscript(YouTubeTranscriptApiException):
"""
Raised if a transcript could not be retrieved.
"""
ERROR_MESSAGE = "\nCould not retrieve a transcript for the video {video_url}!"
CAUSE_MESSAGE_INTRO = " This is most likely caused by:\n\n{cause}"
CAUSE_MESSAGE = ""
GITHUB_REFERRAL = (
"\n\nIf you are sure that the described cause is not responsible for this error "
"and that a transcript should be retrievable, please create an issue at "
"https://github.com/jdepoix/youtube-transcript-api/issues. "
"Please add which version of youtube_transcript_api you are using "
"and provide the information needed to replicate the error. "
"Also make sure that there are no open issues which already describe your problem!"
)
def __init__(self, video_id: str):
self.video_id = video_id
super().__init__()
def _build_error_message(self) -> str:
error_message = self.ERROR_MESSAGE.format(
video_url=WATCH_URL.format(video_id=self.video_id)
)
cause = self.cause
if cause:
error_message += (
self.CAUSE_MESSAGE_INTRO.format(cause=cause) + self.GITHUB_REFERRAL
)
return error_message
@property
def cause(self) -> str:
return self.CAUSE_MESSAGE
def __str__(self) -> str:
return self._build_error_message()
class YouTubeDataUnparsable(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = (
"The data required to fetch the transcript is not parsable. This should "
"not happen, please open an issue (make sure to include the video ID)!"
)
class YouTubeRequestFailed(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = "Request to YouTube failed: {reason}"
def __init__(self, video_id: str, http_error: HTTPError):
self.reason = str(http_error)
super().__init__(video_id)
@property
def cause(self) -> str:
return self.CAUSE_MESSAGE.format(
reason=self.reason,
)
class VideoUnplayable(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = "The video is unplayable for the following reason: {reason}"
SUBREASON_MESSAGE = "\n\nAdditional Details:\n{sub_reasons}"
def __init__(self, video_id: str, reason: Optional[str], sub_reasons: List[str]):
self.reason = reason
self.sub_reasons = sub_reasons
super().__init__(video_id)
@property
def cause(self):
reason = "No reason specified!" if self.reason is None else self.reason
if self.sub_reasons:
sub_reasons = "\n".join(
f" - {sub_reason}" for sub_reason in self.sub_reasons
)
reason = f"{reason}{self.SUBREASON_MESSAGE.format(sub_reasons=sub_reasons)}"
return self.CAUSE_MESSAGE.format(
reason=reason,
)
class VideoUnavailable(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = "The video is no longer available"
class InvalidVideoId(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = (
"You provided an invalid video id. Make sure you are using the video id and NOT the url!\n\n"
'Do NOT run: `YouTubeTranscriptApi().fetch("https://www.youtube.com/watch?v=1234")`\n'
'Instead run: `YouTubeTranscriptApi().fetch("1234")`'
)
class RequestBlocked(CouldNotRetrieveTranscript):
BASE_CAUSE_MESSAGE = (
"YouTube is blocking requests from your IP. This usually is due to one of the "
"following reasons:\n"
"- You have done too many requests and your IP has been blocked by YouTube\n"
"- You are doing requests from an IP belonging to a cloud provider (like AWS, "
"Google Cloud Platform, Azure, etc.). Unfortunately, most IPs from cloud "
"providers are blocked by YouTube.\n\n"
)
CAUSE_MESSAGE = (
f"{BASE_CAUSE_MESSAGE}"
"There are two things you can do to work around this:\n"
'1. Use proxies to hide your IP address, as explained in the "Working around '
'IP bans" section of the README '
"(https://github.com/jdepoix/youtube-transcript-api"
"?tab=readme-ov-file"
"#working-around-ip-bans-requestblocked-or-ipblocked-exception).\n"
"2. (NOT RECOMMENDED) If you authenticate your requests using cookies, you "
"will be able to continue doing requests for a while. However, YouTube will "
"eventually permanently ban the account that you have used to authenticate "
"with! So only do this if you don't mind your account being banned!"
)
WITH_GENERIC_PROXY_CAUSE_MESSAGE = (
"YouTube is blocking your requests, despite you using proxies. Keep in mind "
"that a proxy is just a way to hide your real IP behind the IP of that proxy, "
"but there is no guarantee that the IP of that proxy won't be blocked as "
"well.\n\n"
"The only truly reliable way to prevent IP blocks is rotating through a large "
"pool of residential IPs, by using a provider like Webshare "
"(https://www.webshare.io/?referral_code=w0xno53eb50g), which provides you "
"with a pool of >30M residential IPs (make sure to purchase "
'"Residential" proxies, NOT "Proxy Server" or "Static Residential"!).\n\n'
"You will find more information on how to easily integrate Webshare here: "
"https://github.com/jdepoix/youtube-transcript-api"
"?tab=readme-ov-file#using-webshare"
)
WITH_WEBSHARE_PROXY_CAUSE_MESSAGE = (
"YouTube is blocking your requests, despite you using Webshare proxies. "
'Please make sure that you have purchased "Residential" proxies and '
'NOT "Proxy Server" or "Static Residential", as those won\'t work as '
'reliably! The free tier also uses "Proxy Server" and will NOT work!\n\n'
'The only reliable option is using "Residential" proxies (not "Static '
'Residential"), as this allows you to rotate through a pool of over 30M IPs, '
"which means you will always find an IP that hasn't been blocked by YouTube "
"yet!\n\n"
"You can support the development of this open source project by making your "
"Webshare purchases through this affiliate link: "
"https://www.webshare.io/?referral_code=w0xno53eb50g \n\n"
"Thank you for your support! <3"
)
def __init__(self, video_id: str):
self._proxy_config = None
super().__init__(video_id)
def with_proxy_config(
self, proxy_config: Optional[ProxyConfig]
) -> "RequestBlocked":
self._proxy_config = proxy_config
return self
@property
def cause(self) -> str:
if isinstance(self._proxy_config, WebshareProxyConfig):
return self.WITH_WEBSHARE_PROXY_CAUSE_MESSAGE
if isinstance(self._proxy_config, GenericProxyConfig):
return self.WITH_GENERIC_PROXY_CAUSE_MESSAGE
return super().cause
class IpBlocked(RequestBlocked):
CAUSE_MESSAGE = (
f"{RequestBlocked.BASE_CAUSE_MESSAGE}"
'Ways to work around this are explained in the "Working around IP '
'bans" section of the README (https://github.com/jdepoix/youtube-transcript-api'
"?tab=readme-ov-file"
"#working-around-ip-bans-requestblocked-or-ipblocked-exception).\n"
)
class TranscriptsDisabled(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = "Subtitles are disabled for this video"
class AgeRestricted(CouldNotRetrieveTranscript):
# CAUSE_MESSAGE = (
# "This video is age-restricted. Therefore, you will have to authenticate to be "
# "able to retrieve transcripts for it. You will have to provide a cookie to "
# 'authenticate yourself, as explained in the "Cookie Authentication" section of '
# "the README (https://github.com/jdepoix/youtube-transcript-api"
# "?tab=readme-ov-file#cookie-authentication)"
# )
CAUSE_MESSAGE = (
"This video is age-restricted. Therefore, you are unable to retrieve "
"transcripts for it without authenticating yourself.\n\n"
"Unfortunately, Cookie Authentication is temporarily unsupported in "
"youtube-transcript-api, as recent changes in YouTube's API broke the previous "
"implementation. I will do my best to re-implement it as soon as possible."
)
class NotTranslatable(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = "The requested language is not translatable"
class TranslationLanguageNotAvailable(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = "The requested translation language is not available"
class FailedToCreateConsentCookie(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = "Failed to automatically give consent to saving cookies"
class NoTranscriptFound(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = (
"No transcripts were found for any of the requested language codes: {requested_language_codes}\n\n"
"{transcript_data}"
)
def __init__(
self,
video_id: str,
requested_language_codes: Iterable[str],
transcript_data: "TranscriptList", # noqa: F821
):
self._requested_language_codes = requested_language_codes
self._transcript_data = transcript_data
super().__init__(video_id)
@property
def cause(self) -> str:
return self.CAUSE_MESSAGE.format(
requested_language_codes=self._requested_language_codes,
transcript_data=str(self._transcript_data),
)
class PoTokenRequired(CouldNotRetrieveTranscript):
CAUSE_MESSAGE = (
"The requested video cannot be retrieved without a PO Token. If this happens, "
"please open a GitHub issue!"
)
@@ -0,0 +1,3 @@
WATCH_URL = "https://www.youtube.com/watch?v={video_id}"
INNERTUBE_API_URL = "https://www.youtube.com/youtubei/v1/player?key={api_key}"
INNERTUBE_CONTEXT = {"client": {"clientName": "ANDROID", "clientVersion": "20.10.38"}}
@@ -0,0 +1,492 @@
from dataclasses import dataclass, asdict
from enum import Enum
from itertools import chain
from html import unescape
from typing import List, Dict, Iterator, Iterable, Pattern, Optional
from defusedxml import ElementTree
import re
from requests import HTTPError, Session, Response
from .proxies import ProxyConfig
from ._settings import WATCH_URL, INNERTUBE_CONTEXT, INNERTUBE_API_URL
from ._errors import (
VideoUnavailable,
YouTubeRequestFailed,
NoTranscriptFound,
TranscriptsDisabled,
NotTranslatable,
TranslationLanguageNotAvailable,
FailedToCreateConsentCookie,
InvalidVideoId,
IpBlocked,
RequestBlocked,
AgeRestricted,
VideoUnplayable,
YouTubeDataUnparsable,
PoTokenRequired,
)
@dataclass
class FetchedTranscriptSnippet:
text: str
start: float
"""
The timestamp at which this transcript snippet appears on screen in seconds.
"""
duration: float
"""
The duration of how long the snippet in seconds. Be aware that this is not the
duration of the transcribed speech, but how long the snippet stays on screen.
Therefore, there can be overlaps between snippets!
"""
@dataclass
class FetchedTranscript:
"""
Represents a fetched transcript. This object is iterable, which allows you to
iterate over the transcript snippets.
"""
snippets: List[FetchedTranscriptSnippet]
video_id: str
language: str
language_code: str
is_generated: bool
def __iter__(self) -> Iterator[FetchedTranscriptSnippet]:
return iter(self.snippets)
def __getitem__(self, index) -> FetchedTranscriptSnippet:
return self.snippets[index]
def __len__(self) -> int:
return len(self.snippets)
def to_raw_data(self) -> List[Dict]:
return [asdict(snippet) for snippet in self]
@dataclass
class _TranslationLanguage:
language: str
language_code: str
class _PlayabilityStatus(str, Enum):
OK = "OK"
ERROR = "ERROR"
LOGIN_REQUIRED = "LOGIN_REQUIRED"
class _PlayabilityFailedReason(str, Enum):
BOT_DETECTED = "Sign in to confirm you’re not a bot"
AGE_RESTRICTED = "This video may be inappropriate for some users."
VIDEO_UNAVAILABLE = "This video is unavailable"
def _raise_http_errors(response: Response, video_id: str) -> Response:
try:
if response.status_code == 429:
raise IpBlocked(video_id)
response.raise_for_status()
return response
except HTTPError as error:
raise YouTubeRequestFailed(video_id, error)
class Transcript:
def __init__(
self,
http_client: Session,
video_id: str,
url: str,
language: str,
language_code: str,
is_generated: bool,
translation_languages: List[_TranslationLanguage],
):
"""
You probably don't want to initialize this directly. Usually you'll access Transcript objects using a
TranscriptList.
"""
self._http_client = http_client
self.video_id = video_id
self._url = url
self.language = language
self.language_code = language_code
self.is_generated = is_generated
self.translation_languages = translation_languages
self._translation_languages_dict = {
translation_language.language_code: translation_language.language
for translation_language in translation_languages
}
def fetch(self, preserve_formatting: bool = False) -> FetchedTranscript:
"""
Loads the actual transcript data.
:param preserve_formatting: whether to keep select HTML text formatting
"""
if "&exp=xpe" in self._url:
raise PoTokenRequired(self.video_id)
response = self._http_client.get(self._url)
snippets = _TranscriptParser(preserve_formatting=preserve_formatting).parse(
_raise_http_errors(response, self.video_id).text,
)
return FetchedTranscript(
snippets=snippets,
video_id=self.video_id,
language=self.language,
language_code=self.language_code,
is_generated=self.is_generated,
)
def __str__(self) -> str:
return '{language_code} ("{language}"){translation_description}'.format(
language=self.language,
language_code=self.language_code,
translation_description="[TRANSLATABLE]" if self.is_translatable else "",
)
@property
def is_translatable(self) -> bool:
return len(self.translation_languages) > 0
def translate(self, language_code: str) -> "Transcript":
if not self.is_translatable:
raise NotTranslatable(self.video_id)
if language_code not in self._translation_languages_dict:
raise TranslationLanguageNotAvailable(self.video_id)
return Transcript(
self._http_client,
self.video_id,
"{url}&tlang={language_code}".format(
url=self._url, language_code=language_code
),
self._translation_languages_dict[language_code],
language_code,
True,
[],
)
class TranscriptList:
"""
This object represents a list of transcripts. It can be iterated over to list all transcripts which are available
for a given YouTube video. Also, it provides functionality to search for a transcript in a given language.
"""
def __init__(
self,
video_id: str,
manually_created_transcripts: Dict[str, Transcript],
generated_transcripts: Dict[str, Transcript],
translation_languages: List[_TranslationLanguage],
):
"""
The constructor is only for internal use. Use the static build method instead.
:param video_id: the id of the video this TranscriptList is for
:param manually_created_transcripts: dict mapping language codes to the manually created transcripts
:param generated_transcripts: dict mapping language codes to the generated transcripts
:param translation_languages: list of languages which can be used for translatable languages
"""
self.video_id = video_id
self._manually_created_transcripts = manually_created_transcripts
self._generated_transcripts = generated_transcripts
self._translation_languages = translation_languages
@staticmethod
def build(
http_client: Session, video_id: str, captions_json: Dict
) -> "TranscriptList":
"""
Factory method for TranscriptList.
:param http_client: http client which is used to make the transcript retrieving http calls
:param video_id: the id of the video this TranscriptList is for
:param captions_json: the JSON parsed from the YouTube pages static HTML
:return: the created TranscriptList
"""
translation_languages = [
_TranslationLanguage(
language=translation_language["languageName"]["runs"][0]["text"],
language_code=translation_language["languageCode"],
)
for translation_language in captions_json.get("translationLanguages", [])
]
manually_created_transcripts = {}
generated_transcripts = {}
for caption in captions_json["captionTracks"]:
if caption.get("kind", "") == "asr":
transcript_dict = generated_transcripts
else:
transcript_dict = manually_created_transcripts
transcript_dict[caption["languageCode"]] = Transcript(
http_client,
video_id,
caption["baseUrl"].replace("&fmt=srv3", ""),
caption["name"]["runs"][0]["text"],
caption["languageCode"],
caption.get("kind", "") == "asr",
translation_languages if caption.get("isTranslatable", False) else [],
)
return TranscriptList(
video_id,
manually_created_transcripts,
generated_transcripts,
translation_languages,
)
def __iter__(self) -> Iterator[Transcript]:
return chain(
self._manually_created_transcripts.values(),
self._generated_transcripts.values(),
)
def find_transcript(self, language_codes: Iterable[str]) -> Transcript:
"""
Finds a transcript for a given language code. Manually created transcripts are returned first and only if none
are found, generated transcripts are used. If you only want generated transcripts use
`find_manually_created_transcript` instead.
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
it fails to do so.
:return: the found Transcript
"""
return self._find_transcript(
language_codes,
[self._manually_created_transcripts, self._generated_transcripts],
)
def find_generated_transcript(self, language_codes: Iterable[str]) -> Transcript:
"""
Finds an automatically generated transcript for a given language code.
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
it fails to do so.
:return: the found Transcript
"""
return self._find_transcript(language_codes, [self._generated_transcripts])
def find_manually_created_transcript(
self, language_codes: Iterable[str]
) -> Transcript:
"""
Finds a manually created transcript for a given language code.
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
it fails to do so.
:return: the found Transcript
"""
return self._find_transcript(
language_codes, [self._manually_created_transcripts]
)
def _find_transcript(
self,
language_codes: Iterable[str],
transcript_dicts: List[Dict[str, Transcript]],
) -> Transcript:
for language_code in language_codes:
for transcript_dict in transcript_dicts:
if language_code in transcript_dict:
return transcript_dict[language_code]
raise NoTranscriptFound(self.video_id, language_codes, self)
def __str__(self) -> str:
return (
"For this video ({video_id}) transcripts are available in the following languages:\n\n"
"(MANUALLY CREATED)\n"
"{available_manually_created_transcript_languages}\n\n"
"(GENERATED)\n"
"{available_generated_transcripts}\n\n"
"(TRANSLATION LANGUAGES)\n"
"{available_translation_languages}"
).format(
video_id=self.video_id,
available_manually_created_transcript_languages=self._get_language_description(
str(transcript)
for transcript in self._manually_created_transcripts.values()
),
available_generated_transcripts=self._get_language_description(
str(transcript) for transcript in self._generated_transcripts.values()
),
available_translation_languages=self._get_language_description(
'{language_code} ("{language}")'.format(
language=translation_language.language,
language_code=translation_language.language_code,
)
for translation_language in self._translation_languages
),
)
def _get_language_description(self, transcript_strings: Iterable[str]) -> str:
description = "\n".join(
" - {transcript}".format(transcript=transcript)
for transcript in transcript_strings
)
return description if description else "None"
class TranscriptListFetcher:
def __init__(self, http_client: Session, proxy_config: Optional[ProxyConfig]):
self._http_client = http_client
self._proxy_config = proxy_config
def fetch(self, video_id: str) -> TranscriptList:
return TranscriptList.build(
self._http_client,
video_id,
self._fetch_captions_json(video_id),
)
def _fetch_captions_json(self, video_id: str, try_number: int = 0) -> Dict:
try:
html = self._fetch_video_html(video_id)
api_key = self._extract_innertube_api_key(html, video_id)
innertube_data = self._fetch_innertube_data(video_id, api_key)
return self._extract_captions_json(innertube_data, video_id)
except RequestBlocked as exception:
retries = (
0
if self._proxy_config is None
else self._proxy_config.retries_when_blocked
)
if try_number + 1 < retries:
return self._fetch_captions_json(video_id, try_number=try_number + 1)
raise exception.with_proxy_config(self._proxy_config)
def _extract_innertube_api_key(self, html: str, video_id: str) -> str:
pattern = r'"INNERTUBE_API_KEY":\s*"([a-zA-Z0-9_-]+)"'
match = re.search(pattern, html)
if match and len(match.groups()) == 1:
return match.group(1)
if 'class="g-recaptcha"' in html:
raise IpBlocked(video_id)
raise YouTubeDataUnparsable(video_id) # pragma: no cover
def _extract_captions_json(self, innertube_data: Dict, video_id: str) -> Dict:
self._assert_playability(innertube_data.get("playabilityStatus"), video_id)
captions_json = innertube_data.get("captions", {}).get(
"playerCaptionsTracklistRenderer"
)
if captions_json is None or "captionTracks" not in captions_json:
raise TranscriptsDisabled(video_id)
return captions_json
def _assert_playability(self, playability_status_data: Dict, video_id: str) -> None:
playability_status = playability_status_data.get("status")
if (
playability_status != _PlayabilityStatus.OK.value
and playability_status is not None
):
reason = playability_status_data.get("reason")
if playability_status == _PlayabilityStatus.LOGIN_REQUIRED.value:
if reason == _PlayabilityFailedReason.BOT_DETECTED.value:
raise RequestBlocked(video_id)
if reason == _PlayabilityFailedReason.AGE_RESTRICTED.value:
raise AgeRestricted(video_id)
if (
playability_status == _PlayabilityStatus.ERROR.value
and reason == _PlayabilityFailedReason.VIDEO_UNAVAILABLE.value
):
if video_id.startswith("http://") or video_id.startswith("https://"):
raise InvalidVideoId(video_id)
raise VideoUnavailable(video_id)
subreasons = (
playability_status_data.get("errorScreen", {})
.get("playerErrorMessageRenderer", {})
.get("subreason", {})
.get("runs", [])
)
raise VideoUnplayable(
video_id, reason, [run.get("text", "") for run in subreasons]
)
def _create_consent_cookie(self, html: str, video_id: str) -> None:
match = re.search('name="v" value="(.*?)"', html)
if match is None:
raise FailedToCreateConsentCookie(video_id)
self._http_client.cookies.set(
"CONSENT", "YES+" + match.group(1), domain=".youtube.com"
)
def _fetch_video_html(self, video_id: str) -> str:
html = self._fetch_html(video_id)
if 'action="https://consent.youtube.com/s"' in html:
self._create_consent_cookie(html, video_id)
html = self._fetch_html(video_id)
if 'action="https://consent.youtube.com/s"' in html:
raise FailedToCreateConsentCookie(video_id)
return html
def _fetch_html(self, video_id: str) -> str:
response = self._http_client.get(WATCH_URL.format(video_id=video_id))
return unescape(_raise_http_errors(response, video_id).text)
def _fetch_innertube_data(self, video_id: str, api_key: str) -> Dict:
response = self._http_client.post(
INNERTUBE_API_URL.format(api_key=api_key),
json={
"context": INNERTUBE_CONTEXT,
"videoId": video_id,
},
)
data = _raise_http_errors(response, video_id).json()
return data
class _TranscriptParser:
_FORMATTING_TAGS = [
"strong", # important
"em", # emphasized
"b", # bold
"i", # italic
"mark", # marked
"small", # smaller
"del", # deleted
"ins", # inserted
"sub", # subscript
"sup", # superscript
]
def __init__(self, preserve_formatting: bool = False):
self._html_regex = self._get_html_regex(preserve_formatting)
def _get_html_regex(self, preserve_formatting: bool) -> Pattern[str]:
if preserve_formatting:
formats_regex = "|".join(self._FORMATTING_TAGS)
formats_regex = r"<\/?(?!\/?(" + formats_regex + r")\b).*?\b>"
html_regex = re.compile(formats_regex, re.IGNORECASE)
else:
html_regex = re.compile(r"<[^>]*>", re.IGNORECASE)
return html_regex
def parse(self, raw_data: str) -> List[FetchedTranscriptSnippet]:
return [
FetchedTranscriptSnippet(
text=re.sub(self._html_regex, "", unescape(xml_element.text)),
start=float(xml_element.attrib["start"]),
duration=float(xml_element.attrib.get("dur", "0.0")),
)
for xml_element in ElementTree.fromstring(raw_data)
if xml_element.text is not None
]

Some files were not shown because too many files have changed in this diff Show More