From 41e599f9c31e9e6f136324102f89368520bb9991 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 24 May 2018 09:37:39 -0500 Subject: [PATCH 01/29] Ensure we are utilizing the context created by HTTPSConnection, or falling back to ssl. Fixes #517 --- speedtest.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/speedtest.py b/speedtest.py index 1608eb1..4487a7e 100755 --- a/speedtest.py +++ b/speedtest.py @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.0.1' +__version__ = '2.0.2a' class FakeShutdownEvent(object): @@ -85,9 +85,9 @@ except ImportError: HTTPErrorProcessor, OpenerDirector) try: - from httplib import HTTPConnection + from httplib import HTTPConnection, BadStatusLine except ImportError: - from http.client import HTTPConnection + from http.client import HTTPConnection, BadStatusLine try: from httplib import HTTPSConnection @@ -266,10 +266,13 @@ try: except AttributeError: CERT_ERROR = tuple() - HTTP_ERRORS = ((HTTPError, URLError, socket.error, ssl.SSLError) + - CERT_ERROR) + HTTP_ERRORS = ( + (HTTPError, URLError, socket.error, ssl.SSLError, BadStatusLine) + + CERT_ERROR + ) except ImportError: - HTTP_ERRORS = (HTTPError, URLError, socket.error) + ssl = None + HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine) class SpeedtestException(Exception): @@ -420,14 +423,12 @@ if HTTPSConnection: """ def __init__(self, *args, **kwargs): source_address = kwargs.pop('source_address', None) - context = kwargs.pop('context', None) timeout = kwargs.pop('timeout', 10) HTTPSConnection.__init__(self, *args, **kwargs) - self.source_address = source_address - self._context = context self.timeout = timeout + self.source_address = source_address def connect(self): "Connect to a host on a given (SSL) port." @@ -435,9 +436,13 @@ if HTTPSConnection: SpeedtestHTTPConnection.connect(self) kwargs = {} - if hasattr(ssl, 'SSLContext'): - kwargs['server_hostname'] = self.host - self.sock = self._context.wrap_socket(self.sock, **kwargs) + if ssl: + if hasattr(ssl, 'SSLContext'): + kwargs['server_hostname'] = self.host + try: + self.sock = self._context.wrap_socket(self.sock, **kwargs) + except AttributeError: + self.sock = ssl.wrap_socket(self.sock, **kwargs) def _build_connection(connection, source_address, timeout, context=None): From 72ed585c6f9d19b39a80e0c9364592095d070f62 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 24 May 2018 11:06:29 -0500 Subject: [PATCH 02/29] Bump to v2.0.2 --- speedtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 4487a7e..7a99c99 100755 --- a/speedtest.py +++ b/speedtest.py @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.0.2a' +__version__ = '2.0.2' class FakeShutdownEvent(object): From b2654de4107b236eb5417c1ebeeebbf82a391d71 Mon Sep 17 00:00:00 2001 From: Alex Ward Date: Mon, 3 Dec 2018 16:20:28 +0000 Subject: [PATCH 03/29] Automatically resolve .best property (#514) * automatically call get_best_server * add back SpeedtestBestServerFailer exception --- speedtest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/speedtest.py b/speedtest.py index 7a99c99..276cf3f 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1041,10 +1041,7 @@ class Speedtest(object): @property def best(self): if not self._best: - raise SpeedtestMissingBestServer( - 'get_best_server not called or not able to determine best ' - 'server' - ) + self.get_best_server() return self._best def get_config(self): From a8a32650015997f7847f2de72a29ff7906f53d8a Mon Sep 17 00:00:00 2001 From: liuxu Date: Sat, 1 Dec 2018 23:58:38 +0800 Subject: [PATCH 04/29] Fix python3 upload problem In python3, if Content-length is not set,urllib.request.AbstractHTTPHandler::do_request_() will use "Transfer-encoding:chunked", which will cause HTTPUploader not to exit until timeout --- speedtest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 276cf3f..14d2dd6 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1522,9 +1522,11 @@ class Speedtest(object): ) if pre_allocate: data.pre_allocate() + + headers = {'Content-length': size} requests.append( ( - build_request(self.best['url'], data, secure=self._secure), + build_request(self.best['url'], data, secure=self._secure, headers=headers), size ) ) From 72bf53affa81d1fac97e4e3ca97ed98d62d4a203 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 3 Dec 2018 10:44:49 -0600 Subject: [PATCH 05/29] Fix linting error --- speedtest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 14d2dd6..d44ff5d 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1526,7 +1526,8 @@ class Speedtest(object): headers = {'Content-length': size} requests.append( ( - build_request(self.best['url'], data, secure=self._secure, headers=headers), + build_request(self.best['url'], data, secure=self._secure, + headers=headers), size ) ) From ddb8db0c9458a8ce7ee3c9e82758fd1f943099b1 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 2 Jan 2019 09:18:21 -0600 Subject: [PATCH 06/29] Fix install instructions with git clone. Fixes #566 --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3e57422..e7af2b7 100644 --- a/README.rst +++ b/README.rst @@ -51,7 +51,8 @@ or :: git clone https://github.com/sivel/speedtest-cli.git - python speedtest-cli/setup.py install + cd speedtest-cli + python setup.py install Just download (Like the way it used to be) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From ca2250f700e1e9c70d3c97fa5d091c69bbae7a9f Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 23 Jan 2019 11:33:30 -0600 Subject: [PATCH 07/29] Add functionality for single threaded testing. Fixes #571 --- speedtest.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/speedtest.py b/speedtest.py index d44ff5d..a18435a 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1436,8 +1436,12 @@ class Speedtest(object): printer('Best Server:\n%r' % best, debug=True) return best - def download(self, callback=do_nothing): - """Test download speed against speedtest.net""" + def download(self, callback=do_nothing, threads=None): + """Test download speed against speedtest.net + + A ``threads`` value of ``None`` will fall back to those dictated + by the speedtest.net configuration + """ urls = [] for size in self.config['sizes']['download']: @@ -1476,7 +1480,7 @@ class Speedtest(object): finished.append(sum(thread.result)) callback(thread.i, request_count, end=True) - q = Queue(self.config['threads']['download']) + q = Queue(threads or self.config['threads']['download']) prod_thread = threading.Thread(target=producer, args=(q, requests, request_count)) cons_thread = threading.Thread(target=consumer, @@ -1498,8 +1502,12 @@ class Speedtest(object): self.config['threads']['upload'] = 8 return self.results.download - def upload(self, callback=do_nothing, pre_allocate=True): - """Test upload speed against speedtest.net""" + def upload(self, callback=do_nothing, pre_allocate=True, threads=None): + """Test upload speed against speedtest.net + + A ``threads`` value of ``None`` will fall back to those dictated + by the speedtest.net configuration + """ sizes = [] @@ -1557,7 +1565,7 @@ class Speedtest(object): finished.append(thread.result) callback(thread.i, request_count, end=True) - q = Queue(self.config['threads']['upload']) + q = Queue(threads or self.config['threads']['upload']) prod_thread = threading.Thread(target=producer, args=(q, requests, request_count)) cons_thread = threading.Thread(target=consumer, @@ -1625,6 +1633,10 @@ def parse_args(): parser.add_argument('--no-upload', dest='upload', default=True, action='store_const', const=False, help='Do not perform upload test') + parser.add_argument('--single', default=False, action='store_true', + help='Only use a single connection instead of ' + 'multiple. This simulates a typical file ' + 'transfer.') parser.add_argument('--bytes', dest='units', action='store_const', const=('byte', 8), default=('bit', 1), help='Display values in bytes instead of bits. Does ' @@ -1839,7 +1851,10 @@ def shell(): if args.download: printer('Testing download speed', quiet, end=('', '\n')[bool(debug)]) - speedtest.download(callback=callback) + speedtest.download( + callback=callback, + threads=(None, 1)[args.single] + ) printer('Download: %0.2f M%s/s' % ((results.download / 1000.0 / 1000.0) / args.units[1], args.units[0]), @@ -1850,7 +1865,11 @@ def shell(): if args.upload: printer('Testing upload speed', quiet, end=('', '\n')[bool(debug)]) - speedtest.upload(callback=callback, pre_allocate=args.pre_allocate) + speedtest.upload( + callback=callback, + pre_allocate=args.pre_allocate, + threads=(None, 1)[args.single] + ) printer('Upload: %0.2f M%s/s' % ((results.upload / 1000.0 / 1000.0) / args.units[1], args.units[0]), From 9ac1091eae4d454354337314ef2b3594b75e9b81 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 23 Jan 2019 11:34:00 -0600 Subject: [PATCH 08/29] Add debug support to show if a URL request resulted in a redirect --- speedtest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/speedtest.py b/speedtest.py index a18435a..dcda09c 100755 --- a/speedtest.py +++ b/speedtest.py @@ -666,6 +666,8 @@ def catch_request(request, opener=None): try: uh = _open(request) + if request.get_full_url() != uh.geturl(): + printer('Redirected to %s' % uh.geturl(), debug=True) return uh, False except HTTP_ERRORS: e = get_exception() From b0b826c8703797eb2011ab70d2a2b6ad8f4a08e2 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 23 Jan 2019 11:34:23 -0600 Subject: [PATCH 09/29] Add the python version to the version output --- speedtest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index dcda09c..17ba7d9 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1602,7 +1602,8 @@ def ctrl_c(shutdown_event): def version(): """Print the version""" - printer(__version__) + printer('speedtest-cli %s' % __version__) + printer('Python %s' % sys.version.replace('\n', '')) sys.exit(0) From b43334f1ec7143e5ddf993a33d6a394bb4f2535d Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 19 Feb 2019 16:37:23 -0600 Subject: [PATCH 10/29] Switch from platform.system to platform.platform. Fixes #574 --- speedtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 17ba7d9..ff0eabf 100755 --- a/speedtest.py +++ b/speedtest.py @@ -607,7 +607,7 @@ def build_user_agent(): ua_tuple = ( 'Mozilla/5.0', - '(%s; U; %s; en-us)' % (platform.system(), platform.architecture()[0]), + '(%s; U; %s; en-us)' % (platform.platform(), platform.architecture()[0]), 'Python/%s' % platform.python_version(), '(KHTML, like Gecko)', 'speedtest-cli/%s' % __version__ From 217ce8eff1a1ce307b047d86793a4593890cfa22 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 19 Feb 2019 16:56:26 -0600 Subject: [PATCH 11/29] ssl.wrap_socket doesn't support server_hostname. See #572 --- speedtest.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/speedtest.py b/speedtest.py index ff0eabf..3f58e6f 100755 --- a/speedtest.py +++ b/speedtest.py @@ -435,14 +435,18 @@ if HTTPSConnection: SpeedtestHTTPConnection.connect(self) - kwargs = {} if ssl: - if hasattr(ssl, 'SSLContext'): - kwargs['server_hostname'] = self.host try: + kwargs = {} + if hasattr(ssl, 'SSLContext'): + kwargs['server_hostname'] = self.host self.sock = self._context.wrap_socket(self.sock, **kwargs) except AttributeError: - self.sock = ssl.wrap_socket(self.sock, **kwargs) + self.sock = ssl.wrap_socket(self.sock) + try: + self.sock.server_hostname = self.host + except AttributeError: + pass def _build_connection(connection, source_address, timeout, context=None): From 6cf43b2ff7fe67f63351b7e537b9944ea434d76e Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 19 Feb 2019 16:58:50 -0600 Subject: [PATCH 12/29] linting fix --- speedtest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 3f58e6f..464e2f7 100755 --- a/speedtest.py +++ b/speedtest.py @@ -611,7 +611,8 @@ def build_user_agent(): ua_tuple = ( 'Mozilla/5.0', - '(%s; U; %s; en-us)' % (platform.platform(), platform.architecture()[0]), + '(%s; U; %s; en-us)' % (platform.platform(), + platform.architecture()[0]), 'Python/%s' % platform.python_version(), '(KHTML, like Gecko)', 'speedtest-cli/%s' % __version__ From f356c7b02d7941a8f114caeefe63ae2747152cf2 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 19 Feb 2019 17:17:25 -0600 Subject: [PATCH 13/29] ensure ERROR doesn't print an empty string --- speedtest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 464e2f7..217ee39 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1915,7 +1915,10 @@ def main(): e = get_exception() # Ignore a successful exit, or argparse exit if getattr(e, 'code', 1) not in (0, 2): - raise SystemExit('ERROR: %s' % e) + msg = '%s' % e + if not msg: + msg = '%r' % e + raise SystemExit('ERROR: %s' % msg) if __name__ == '__main__': From fb0569946d5a1295f3bff59233c4834f78322e13 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 5 Mar 2019 10:55:57 -0600 Subject: [PATCH 14/29] Bump to 2.1.0 for upcoming release --- speedtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 217ee39..a011075 100755 --- a/speedtest.py +++ b/speedtest.py @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.0.2' +__version__ = '2.1.0' class FakeShutdownEvent(object): From 69ddff1a11331d20ba5877abe6491082e90e1726 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 5 Mar 2019 11:44:19 -0600 Subject: [PATCH 15/29] Disable py2.4/2.5 tests for now --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0380352..dc2c5fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ addons: sources: - deadsnakes packages: - - python2.4 - - python2.5 + # - python2.4 + # - python2.5 - python2.6 - python3.2 - python3.3 @@ -15,10 +15,10 @@ matrix: include: - python: 2.7 env: TOXENV=flake8 - - python: 2.7 - env: TOXENV=py24 - - python: 2.7 - env: TOXENV=py25 + # - python: 2.7 + # env: TOXENV=py24 + # - python: 2.7 + # env: TOXENV=py25 - python: 2.7 env: TOXENV=py26 - python: 2.7 From 3109fcf407a948f1a867da122267cfe9ece7150f Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 11 Mar 2019 09:57:19 -0500 Subject: [PATCH 16/29] Update usage --- README.rst | 18 ++++++++++-------- speedtest-cli.1 | 5 +++++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index e7af2b7..7a98f31 100644 --- a/README.rst +++ b/README.rst @@ -75,21 +75,23 @@ Usage :: $ speedtest-cli -h - usage: speedtest-cli [-h] [--no-download] [--no-upload] [--bytes] [--share] - [--simple] [--csv] [--csv-delimiter CSV_DELIMITER] - [--csv-header] [--json] [--list] [--server SERVER] - [--exclude EXCLUDE] [--mini MINI] [--source SOURCE] - [--timeout TIMEOUT] [--secure] [--no-pre-allocate] - [--version] - + usage: speedtest-cli [-h] [--no-download] [--no-upload] [--single] [--bytes] + [--share] [--simple] [--csv] + [--csv-delimiter CSV_DELIMITER] [--csv-header] [--json] + [--list] [--server SERVER] [--exclude EXCLUDE] + [--mini MINI] [--source SOURCE] [--timeout TIMEOUT] + [--secure] [--no-pre-allocate] [--version] + Command line interface for testing internet bandwidth using speedtest.net. -------------------------------------------------------------------------- https://github.com/sivel/speedtest-cli - + optional arguments: -h, --help show this help message and exit --no-download Do not perform download test --no-upload Do not perform upload test + --single Only use a single connection instead of multiple. This + simulates a typical file transfer. --bytes Display values in bytes instead of bits. Does not affect the image generated by --share, nor output from --json or --csv diff --git a/speedtest-cli.1 b/speedtest-cli.1 index 87558b4..367c977 100644 --- a/speedtest-cli.1 +++ b/speedtest-cli.1 @@ -33,6 +33,11 @@ Do not perform download test Do not perform upload test .RE +\fB\-\-single\fR +.RS +Only use a single connection instead of multiple. This simulates a typical file transfer. +.RE + \fB\-\-bytes\fR .RS Display values in bytes instead of bits. Does not affect the image generated by \-\-share, nor output from \-\-json or \-\-csv From 2d5a9ef364258621745edafb69c8e592982dac21 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 11 Mar 2019 10:03:12 -0500 Subject: [PATCH 17/29] Switch copyright from range, to date started --- setup.py | 2 +- speedtest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 29e0f31..a617be4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2012-2018 Matt Martz +# Copyright 2012 Matt Martz # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/speedtest.py b/speedtest.py index a011075..be93123 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2012-2018 Matt Martz +# Copyright 2012 Matt Martz # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may From 9af203652b286ed320bc21ed2f5378317045ec77 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 12 Mar 2019 10:55:23 -0500 Subject: [PATCH 18/29] Python2.4/2.5 SSL support --- speedtest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/speedtest.py b/speedtest.py index be93123..475a96d 100755 --- a/speedtest.py +++ b/speedtest.py @@ -97,6 +97,11 @@ except ImportError: except ImportError: HTTPSConnection = None +try: + from httplib import FakeSocket +except ImportError: + FakeSocket = None + try: from Queue import Queue except ImportError: @@ -447,6 +452,20 @@ if HTTPSConnection: self.sock.server_hostname = self.host except AttributeError: pass + elif FakeSocket: + # Python 2.4/2.5 support + try: + self.sock = FakeSocket(self.sock, socket.ssl(self.sock)) + except AttributeError: + raise SpeedtestException( + 'This version of Python does not support HTTPS/SSL ' + 'functionality' + ) + else: + raise SpeedtestException( + 'This version of Python does not support HTTPS/SSL ' + 'functionality' + ) def _build_connection(connection, source_address, timeout, context=None): From cdf60028659ad046b284a3a04792ddc1b760aaa6 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 12 Mar 2019 10:58:17 -0500 Subject: [PATCH 19/29] Bump to 2.1.1 --- speedtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 475a96d..e0d15e9 100755 --- a/speedtest.py +++ b/speedtest.py @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.1.0' +__version__ = '2.1.1' class FakeShutdownEvent(object): From 681cdf20a5fe774ba3aefedd0cd88cbff5384439 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 12 Mar 2019 11:01:31 -0500 Subject: [PATCH 20/29] Re-enable python 2.4 and 2.5 testing --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc2c5fc..0380352 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ addons: sources: - deadsnakes packages: - # - python2.4 - # - python2.5 + - python2.4 + - python2.5 - python2.6 - python3.2 - python3.3 @@ -15,10 +15,10 @@ matrix: include: - python: 2.7 env: TOXENV=flake8 - # - python: 2.7 - # env: TOXENV=py24 - # - python: 2.7 - # env: TOXENV=py25 + - python: 2.7 + env: TOXENV=py24 + - python: 2.7 + env: TOXENV=py25 - python: 2.7 env: TOXENV=py26 - python: 2.7 From 81bba6070c9069f8770212f119232d0622ddb111 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 13 Mar 2019 15:56:00 -0500 Subject: [PATCH 21/29] Add support for py38 without deprecation warnings (#585) * Add support for py38 without deprecation warnings * Address Py2.5 issue * Add py3.7 and 3.8 * xenial * pypy trusty --- .travis.yml | 7 +++++++ speedtest.py | 44 +++++++++++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0380352..c40859c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ language: python +sudo: required +dist: xenial addons: apt: @@ -33,8 +35,13 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 + - python: 3.7 + env: TOXENV=py37 + - python: 3.8-dev + env: TOXENV=py38 - python: pypy env: TOXENV=pypy + dist: trusty before_install: - if [[ $(echo "$TOXENV" | egrep -c "py35") != 0 ]]; then pyenv global system 3.5; fi; diff --git a/speedtest.py b/speedtest.py index e0d15e9..17c52a2 100755 --- a/speedtest.py +++ b/speedtest.py @@ -53,6 +53,9 @@ class FakeShutdownEvent(object): # Some global variables we use DEBUG = False _GLOBAL_DEFAULT_TIMEOUT = object() +PY25PLUS = sys.version_info[:2] >= (2, 5) +PY26PLUS = sys.version_info[:2] >= (2, 6) +PY32PLUS = sys.version_info[:2] >= (3, 2) # Begin import game to handle Python 2 and Python 3 try: @@ -64,14 +67,15 @@ except ImportError: json = None try: - import xml.etree.cElementTree as ET -except ImportError: + import xml.etree.ElementTree as ET try: - import xml.etree.ElementTree as ET + from xml.etree.ElementTree import _Element as ET_Element except ImportError: - from xml.dom import minidom as DOM - from xml.parsers.expat import ExpatError - ET = None + pass +except ImportError: + from xml.dom import minidom as DOM + from xml.parsers.expat import ExpatError + ET = None try: from urllib2 import (urlopen, Request, HTTPError, URLError, @@ -262,6 +266,16 @@ else: write(arg) write(end) +if PY32PLUS: + etree_iter = ET.Element.iter +elif PY25PLUS: + etree_iter = ET_Element.getiterator + +if PY26PLUS: + thread_is_alive = threading.Thread.is_alive +else: + thread_is_alive = threading.Thread.isAlive + # Exception "constants" to support Python 2 through Python 3 try: @@ -1262,7 +1276,7 @@ class Speedtest(object): raise SpeedtestServersError( 'Malformed speedtest.net server list: %s' % e ) - elements = root.getiterator('server') + elements = etree_iter(root, 'server') except AttributeError: try: root = DOM.parseString(serversxml) @@ -1499,9 +1513,10 @@ class Speedtest(object): finished = [] def consumer(q, request_count): + _is_alive = thread_is_alive while len(finished) < request_count: thread = q.get(True) - while thread.isAlive(): + while _is_alive(thread): thread.join(timeout=0.1) finished.append(sum(thread.result)) callback(thread.i, request_count, end=True) @@ -1514,9 +1529,10 @@ class Speedtest(object): start = timeit.default_timer() prod_thread.start() cons_thread.start() - while prod_thread.isAlive(): + _is_alive = thread_is_alive + while _is_alive(prod_thread): prod_thread.join(timeout=0.1) - while cons_thread.isAlive(): + while _is_alive(cons_thread): cons_thread.join(timeout=0.1) stop = timeit.default_timer() @@ -1584,9 +1600,10 @@ class Speedtest(object): finished = [] def consumer(q, request_count): + _is_alive = thread_is_alive while len(finished) < request_count: thread = q.get(True) - while thread.isAlive(): + while _is_alive(thread): thread.join(timeout=0.1) finished.append(thread.result) callback(thread.i, request_count, end=True) @@ -1599,9 +1616,10 @@ class Speedtest(object): start = timeit.default_timer() prod_thread.start() cons_thread.start() - while prod_thread.isAlive(): + _is_alive = thread_is_alive + while _is_alive(prod_thread): prod_thread.join(timeout=0.1) - while cons_thread.isAlive(): + while _is_alive(cons_thread): cons_thread.join(timeout=0.1) stop = timeit.default_timer() From 2658bd50b448e98ce5f3688f66b91aaeba5a897a Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 13 Mar 2019 15:57:05 -0500 Subject: [PATCH 22/29] Bump devel version --- speedtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 17c52a2..7be8d65 100755 --- a/speedtest.py +++ b/speedtest.py @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.1.1' +__version__ = '2.1.2a' class FakeShutdownEvent(object): From 7ebb9965ddaeb64ecb27efdf06e91d661c129301 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 21 Aug 2019 12:25:32 -0500 Subject: [PATCH 23/29] Ensure threads don't start before a position in the queue is available. Fixes #628 --- speedtest.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/speedtest.py b/speedtest.py index 7be8d65..b7a5124 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1496,6 +1496,9 @@ class Speedtest(object): build_request(url, bump=i, secure=self._secure) ) + max_threads = threads or self.config['threads']['download'] + in_flight = {'threads': 0} + def producer(q, requests, request_count): for i, request in enumerate(requests): thread = HTTPDownloader( @@ -1506,8 +1509,11 @@ class Speedtest(object): opener=self._opener, shutdown_event=self._shutdown_event ) + while in_flight['threads'] >= max_threads: + timeit.time.sleep(0.001) thread.start() q.put(thread, True) + in_flight['threads'] += 1 callback(i, request_count, start=True) finished = [] @@ -1517,11 +1523,12 @@ class Speedtest(object): while len(finished) < request_count: thread = q.get(True) while _is_alive(thread): - thread.join(timeout=0.1) + thread.join(timeout=0.001) + in_flight['threads'] -= 1 finished.append(sum(thread.result)) callback(thread.i, request_count, end=True) - q = Queue(threads or self.config['threads']['download']) + q = Queue(max_threads) prod_thread = threading.Thread(target=producer, args=(q, requests, request_count)) cons_thread = threading.Thread(target=consumer, @@ -1531,9 +1538,9 @@ class Speedtest(object): cons_thread.start() _is_alive = thread_is_alive while _is_alive(prod_thread): - prod_thread.join(timeout=0.1) + prod_thread.join(timeout=0.001) while _is_alive(cons_thread): - cons_thread.join(timeout=0.1) + cons_thread.join(timeout=0.001) stop = timeit.default_timer() self.results.bytes_received = sum(finished) @@ -1582,6 +1589,9 @@ class Speedtest(object): ) ) + max_threads = threads or self.config['threads']['upload'] + in_flight = {'threads': 0} + def producer(q, requests, request_count): for i, request in enumerate(requests[:request_count]): thread = HTTPUploader( @@ -1593,8 +1603,11 @@ class Speedtest(object): opener=self._opener, shutdown_event=self._shutdown_event ) + while in_flight['threads'] >= max_threads: + timeit.time.sleep(0.001) thread.start() q.put(thread, True) + in_flight['threads'] += 1 callback(i, request_count, start=True) finished = [] @@ -1604,7 +1617,8 @@ class Speedtest(object): while len(finished) < request_count: thread = q.get(True) while _is_alive(thread): - thread.join(timeout=0.1) + thread.join(timeout=0.001) + in_flight['threads'] -= 1 finished.append(thread.result) callback(thread.i, request_count, end=True) From 266e53c25636e0dcee31b599c49113d4d7cf8298 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Fri, 5 Jul 2019 13:47:54 -0500 Subject: [PATCH 24/29] Fix proxy support. Fixes #610 --- speedtest.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/speedtest.py b/speedtest.py index b7a5124..eb131ee 100755 --- a/speedtest.py +++ b/speedtest.py @@ -413,6 +413,8 @@ class SpeedtestHTTPConnection(HTTPConnection): source_address = kwargs.pop('source_address', None) timeout = kwargs.pop('timeout', 10) + self._tunnel_host = None + HTTPConnection.__init__(self, *args, **kwargs) self.source_address = source_address @@ -433,17 +435,23 @@ class SpeedtestHTTPConnection(HTTPConnection): self.source_address ) + if self._tunnel_host: + self._tunnel() + if HTTPSConnection: - class SpeedtestHTTPSConnection(HTTPSConnection, - SpeedtestHTTPConnection): + class SpeedtestHTTPSConnection(HTTPSConnection): """Custom HTTPSConnection to support source_address across Python 2.4 - Python 3 """ + default_port = 443 + def __init__(self, *args, **kwargs): source_address = kwargs.pop('source_address', None) timeout = kwargs.pop('timeout', 10) + self._tunnel_host = None + HTTPSConnection.__init__(self, *args, **kwargs) self.timeout = timeout @@ -451,14 +459,30 @@ if HTTPSConnection: def connect(self): "Connect to a host on a given (SSL) port." + try: + self.sock = socket.create_connection( + (self.host, self.port), + self.timeout, + self.source_address + ) + except (AttributeError, TypeError): + self.sock = create_connection( + (self.host, self.port), + self.timeout, + self.source_address + ) - SpeedtestHTTPConnection.connect(self) + if self._tunnel_host: + self._tunnel() if ssl: try: kwargs = {} if hasattr(ssl, 'SSLContext'): - kwargs['server_hostname'] = self.host + if self._tunnel_host: + kwargs['server_hostname'] = self._tunnel_host + else: + kwargs['server_hostname'] = self.host self.sock = self._context.wrap_socket(self.sock, **kwargs) except AttributeError: self.sock = ssl.wrap_socket(self.sock) From c58ad3367bf27f4b4a4d5b1bca29ebd574731c5d Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 22 Aug 2019 09:48:18 -0500 Subject: [PATCH 25/29] Bump release --- speedtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index eb131ee..92a2be0 100755 --- a/speedtest.py +++ b/speedtest.py @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.1.2a' +__version__ = '2.1.2' class FakeShutdownEvent(object): From db46af8bcd69dd468e57752551e2e344b28dbc91 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 19 Jan 2021 17:04:47 -0600 Subject: [PATCH 26/29] Ensure we catch HTTP errors on upload/download. Fixes #752 --- speedtest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index 92a2be0..425f18c 100755 --- a/speedtest.py +++ b/speedtest.py @@ -817,6 +817,8 @@ class HTTPDownloader(threading.Thread): f.close() except IOError: pass + except HTTP_ERRORS: + pass class HTTPUploaderData(object): @@ -882,7 +884,7 @@ class HTTPUploader(threading.Thread): self.request = request self.request.data.start = self.starttime = start self.size = size - self.result = None + self.result = 0 self.timeout = timeout self.i = i @@ -917,6 +919,8 @@ class HTTPUploader(threading.Thread): self.result = 0 except (IOError, SpeedtestUploadTimeout): self.result = sum(self.request.data.total) + except HTTP_ERRORS: + self.result = 0 class SpeedtestResults(object): From cadc68b5aef20f28648072cf07a8f155639b81dd Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 8 Apr 2021 08:44:32 -0500 Subject: [PATCH 27/29] Handle case where ignoreids is empty or contains empty ids --- speedtest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/speedtest.py b/speedtest.py index 425f18c..a62a184 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1174,9 +1174,9 @@ class Speedtest(object): # times = get_attributes_by_tag_name(root, 'times') client = get_attributes_by_tag_name(root, 'client') - ignore_servers = list( - map(int, server_config['ignoreids'].split(',')) - ) + ignore_servers = [ + int(i) for i in server_config['ignoreids'].split(',') if i + ] ratio = int(upload['ratio']) upload_max = int(upload['maxchunkcount']) From 42e96b13dda2afabbcec2622612d13495a415caa Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 8 Apr 2021 08:45:29 -0500 Subject: [PATCH 28/29] Bump to 2.1.3 --- speedtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speedtest.py b/speedtest.py index a62a184..a33296d 100755 --- a/speedtest.py +++ b/speedtest.py @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.1.2' +__version__ = '2.1.3' class FakeShutdownEvent(object): From 22210ca35228f0bbcef75a7c14587c4ecb875ab4 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 7 Jul 2021 14:50:15 -0500 Subject: [PATCH 29/29] Python 3.10 support --- setup.py | 3 +++ speedtest.py | 55 ++++++++++++++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index a617be4..f3d21ad 100644 --- a/setup.py +++ b/setup.py @@ -92,5 +92,8 @@ setup( 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ] ) diff --git a/speedtest.py b/speedtest.py index a33296d..186b529 100755 --- a/speedtest.py +++ b/speedtest.py @@ -15,18 +15,18 @@ # License for the specific language governing permissions and limitations # under the License. -import os -import re import csv -import sys -import math +import datetime import errno +import math +import os +import platform +import re import signal import socket -import timeit -import datetime -import platform +import sys import threading +import timeit import xml.parsers.expat try: @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.1.3' +__version__ = '2.1.4b1' class FakeShutdownEvent(object): @@ -49,6 +49,8 @@ class FakeShutdownEvent(object): "Dummy method to always return false""" return False + is_set = isSet + # Some global variables we use DEBUG = False @@ -56,6 +58,7 @@ _GLOBAL_DEFAULT_TIMEOUT = object() PY25PLUS = sys.version_info[:2] >= (2, 5) PY26PLUS = sys.version_info[:2] >= (2, 6) PY32PLUS = sys.version_info[:2] >= (3, 2) +PY310PLUS = sys.version_info[:2] >= (3, 10) # Begin import game to handle Python 2 and Python 3 try: @@ -266,17 +269,6 @@ else: write(arg) write(end) -if PY32PLUS: - etree_iter = ET.Element.iter -elif PY25PLUS: - etree_iter = ET_Element.getiterator - -if PY26PLUS: - thread_is_alive = threading.Thread.is_alive -else: - thread_is_alive = threading.Thread.isAlive - - # Exception "constants" to support Python 2 through Python 3 try: import ssl @@ -293,6 +285,23 @@ except ImportError: ssl = None HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine) +if PY32PLUS: + etree_iter = ET.Element.iter +elif PY25PLUS: + etree_iter = ET_Element.getiterator + +if PY26PLUS: + thread_is_alive = threading.Thread.is_alive +else: + thread_is_alive = threading.Thread.isAlive + + +def event_is_set(event): + try: + return event.is_set() + except AttributeError: + return event.isSet() + class SpeedtestException(Exception): """Base exception for this module""" @@ -769,7 +778,7 @@ def print_dots(shutdown_event): status """ def inner(current, total, start=False, end=False): - if shutdown_event.isSet(): + if event_is_set(shutdown_event): return sys.stdout.write('.') @@ -808,7 +817,7 @@ class HTTPDownloader(threading.Thread): try: if (timeit.default_timer() - self.starttime) <= self.timeout: f = self._opener(self.request) - while (not self._shutdown_event.isSet() and + while (not event_is_set(self._shutdown_event) and (timeit.default_timer() - self.starttime) <= self.timeout): self.result.append(len(f.read(10240))) @@ -864,7 +873,7 @@ class HTTPUploaderData(object): def read(self, n=10240): if ((timeit.default_timer() - self.start) <= self.timeout and - not self._shutdown_event.isSet()): + not event_is_set(self._shutdown_event)): chunk = self.data.read(n) self.total.append(len(chunk)) return chunk @@ -902,7 +911,7 @@ class HTTPUploader(threading.Thread): request = self.request try: if ((timeit.default_timer() - self.starttime) <= self.timeout and - not self._shutdown_event.isSet()): + not event_is_set(self._shutdown_event)): try: f = self._opener(request) except TypeError: