diff --git a/.travis.yml b/.travis.yml index c40859c..0380352 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,4 @@ language: python -sudo: required -dist: xenial addons: apt: @@ -35,13 +33,8 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - - python: 3.8-dev - env: TOXENV=py38 - python: pypy env: TOXENV=pypy - dist: trusty before_install: - if [[ $(echo "$TOXENV" | egrep -c "py35") != 0 ]]; then pyenv global system 3.5; fi; diff --git a/README.rst b/README.rst index 7a98f31..3e57422 100644 --- a/README.rst +++ b/README.rst @@ -51,8 +51,7 @@ or :: git clone https://github.com/sivel/speedtest-cli.git - cd speedtest-cli - python setup.py install + python speedtest-cli/setup.py install Just download (Like the way it used to be) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -75,23 +74,21 @@ Usage :: $ speedtest-cli -h - usage: speedtest-cli [-h] [--no-download] [--no-upload] [--single] [--bytes] - [--share] [--simple] [--csv] - [--csv-delimiter CSV_DELIMITER] [--csv-header] [--json] - [--list] [--server SERVER] [--exclude EXCLUDE] - [--mini MINI] [--source SOURCE] [--timeout TIMEOUT] - [--secure] [--no-pre-allocate] [--version] - + usage: speedtest-cli [-h] [--no-download] [--no-upload] [--bytes] [--share] + [--simple] [--csv] [--csv-delimiter CSV_DELIMITER] + [--csv-header] [--json] [--list] [--server SERVER] + [--exclude EXCLUDE] [--mini MINI] [--source SOURCE] + [--timeout TIMEOUT] [--secure] [--no-pre-allocate] + [--version] + Command line interface for testing internet bandwidth using speedtest.net. -------------------------------------------------------------------------- https://github.com/sivel/speedtest-cli - + optional arguments: -h, --help show this help message and exit --no-download Do not perform download test --no-upload Do not perform upload test - --single Only use a single connection instead of multiple. This - simulates a typical file transfer. --bytes Display values in bytes instead of bits. Does not affect the image generated by --share, nor output from --json or --csv diff --git a/setup.py b/setup.py index f3d21ad..29e0f31 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2012 Matt Martz +# Copyright 2012-2018 Matt Martz # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -92,8 +92,5 @@ setup( 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', ] ) diff --git a/speedtest-cli.1 b/speedtest-cli.1 index 367c977..87558b4 100644 --- a/speedtest-cli.1 +++ b/speedtest-cli.1 @@ -33,11 +33,6 @@ Do not perform download test Do not perform upload test .RE -\fB\-\-single\fR -.RS -Only use a single connection instead of multiple. This simulates a typical file transfer. -.RE - \fB\-\-bytes\fR .RS Display values in bytes instead of bits. Does not affect the image generated by \-\-share, nor output from \-\-json or \-\-csv diff --git a/speedtest.py b/speedtest.py index 186b529..1608eb1 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2012 Matt Martz +# Copyright 2012-2018 Matt Martz # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,18 +15,18 @@ # License for the specific language governing permissions and limitations # under the License. -import csv -import datetime -import errno -import math import os -import platform import re +import csv +import sys +import math +import errno import signal import socket -import sys -import threading import timeit +import datetime +import platform +import threading import xml.parsers.expat try: @@ -36,7 +36,7 @@ except ImportError: gzip = None GZIP_BASE = object -__version__ = '2.1.4b1' +__version__ = '2.0.1' class FakeShutdownEvent(object): @@ -49,16 +49,10 @@ class FakeShutdownEvent(object): "Dummy method to always return false""" return False - is_set = isSet - # Some global variables we use DEBUG = False _GLOBAL_DEFAULT_TIMEOUT = object() -PY25PLUS = sys.version_info[:2] >= (2, 5) -PY26PLUS = sys.version_info[:2] >= (2, 6) -PY32PLUS = sys.version_info[:2] >= (3, 2) -PY310PLUS = sys.version_info[:2] >= (3, 10) # Begin import game to handle Python 2 and Python 3 try: @@ -70,15 +64,14 @@ except ImportError: json = None try: - import xml.etree.ElementTree as ET - try: - from xml.etree.ElementTree import _Element as ET_Element - except ImportError: - pass + import xml.etree.cElementTree as ET except ImportError: - from xml.dom import minidom as DOM - from xml.parsers.expat import ExpatError - ET = None + try: + import xml.etree.ElementTree as ET + except ImportError: + from xml.dom import minidom as DOM + from xml.parsers.expat import ExpatError + ET = None try: from urllib2 import (urlopen, Request, HTTPError, URLError, @@ -92,9 +85,9 @@ except ImportError: HTTPErrorProcessor, OpenerDirector) try: - from httplib import HTTPConnection, BadStatusLine + from httplib import HTTPConnection except ImportError: - from http.client import HTTPConnection, BadStatusLine + from http.client import HTTPConnection try: from httplib import HTTPSConnection @@ -104,11 +97,6 @@ except ImportError: except ImportError: HTTPSConnection = None -try: - from httplib import FakeSocket -except ImportError: - FakeSocket = None - try: from Queue import Queue except ImportError: @@ -269,6 +257,7 @@ else: write(arg) write(end) + # Exception "constants" to support Python 2 through Python 3 try: import ssl @@ -277,30 +266,10 @@ try: except AttributeError: CERT_ERROR = tuple() - HTTP_ERRORS = ( - (HTTPError, URLError, socket.error, ssl.SSLError, BadStatusLine) + - CERT_ERROR - ) + HTTP_ERRORS = ((HTTPError, URLError, socket.error, ssl.SSLError) + + CERT_ERROR) except ImportError: - ssl = None - HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine) - -if PY32PLUS: - etree_iter = ET.Element.iter -elif PY25PLUS: - etree_iter = ET_Element.getiterator - -if PY26PLUS: - thread_is_alive = threading.Thread.is_alive -else: - thread_is_alive = threading.Thread.isAlive - - -def event_is_set(event): - try: - return event.is_set() - except AttributeError: - return event.isSet() + HTTP_ERRORS = (HTTPError, URLError, socket.error) class SpeedtestException(Exception): @@ -422,8 +391,6 @@ class SpeedtestHTTPConnection(HTTPConnection): source_address = kwargs.pop('source_address', None) timeout = kwargs.pop('timeout', 10) - self._tunnel_host = None - HTTPConnection.__init__(self, *args, **kwargs) self.source_address = source_address @@ -444,75 +411,33 @@ class SpeedtestHTTPConnection(HTTPConnection): self.source_address ) - if self._tunnel_host: - self._tunnel() - if HTTPSConnection: - class SpeedtestHTTPSConnection(HTTPSConnection): + class SpeedtestHTTPSConnection(HTTPSConnection, + SpeedtestHTTPConnection): """Custom HTTPSConnection to support source_address across Python 2.4 - Python 3 """ - default_port = 443 - def __init__(self, *args, **kwargs): source_address = kwargs.pop('source_address', None) + context = kwargs.pop('context', None) timeout = kwargs.pop('timeout', 10) - self._tunnel_host = None - HTTPSConnection.__init__(self, *args, **kwargs) - self.timeout = timeout self.source_address = source_address + self._context = context + self.timeout = timeout def connect(self): "Connect to a host on a given (SSL) port." - try: - self.sock = socket.create_connection( - (self.host, self.port), - self.timeout, - self.source_address - ) - except (AttributeError, TypeError): - self.sock = create_connection( - (self.host, self.port), - self.timeout, - self.source_address - ) - if self._tunnel_host: - self._tunnel() + SpeedtestHTTPConnection.connect(self) - if ssl: - try: - kwargs = {} - if hasattr(ssl, 'SSLContext'): - if self._tunnel_host: - kwargs['server_hostname'] = self._tunnel_host - else: - kwargs['server_hostname'] = self.host - self.sock = self._context.wrap_socket(self.sock, **kwargs) - except AttributeError: - self.sock = ssl.wrap_socket(self.sock) - try: - self.sock.server_hostname = self.host - except AttributeError: - pass - elif FakeSocket: - # Python 2.4/2.5 support - try: - self.sock = FakeSocket(self.sock, socket.ssl(self.sock)) - except AttributeError: - raise SpeedtestException( - 'This version of Python does not support HTTPS/SSL ' - 'functionality' - ) - else: - raise SpeedtestException( - 'This version of Python does not support HTTPS/SSL ' - 'functionality' - ) + kwargs = {} + if hasattr(ssl, 'SSLContext'): + kwargs['server_hostname'] = self.host + self.sock = self._context.wrap_socket(self.sock, **kwargs) def _build_connection(connection, source_address, timeout, context=None): @@ -677,8 +602,7 @@ def build_user_agent(): ua_tuple = ( 'Mozilla/5.0', - '(%s; U; %s; en-us)' % (platform.platform(), - platform.architecture()[0]), + '(%s; U; %s; en-us)' % (platform.system(), platform.architecture()[0]), 'Python/%s' % platform.python_version(), '(KHTML, like Gecko)', 'speedtest-cli/%s' % __version__ @@ -737,8 +661,6 @@ def catch_request(request, opener=None): try: uh = _open(request) - if request.get_full_url() != uh.geturl(): - printer('Redirected to %s' % uh.geturl(), debug=True) return uh, False except HTTP_ERRORS: e = get_exception() @@ -778,7 +700,7 @@ def print_dots(shutdown_event): status """ def inner(current, total, start=False, end=False): - if event_is_set(shutdown_event): + if shutdown_event.isSet(): return sys.stdout.write('.') @@ -817,7 +739,7 @@ class HTTPDownloader(threading.Thread): try: if (timeit.default_timer() - self.starttime) <= self.timeout: f = self._opener(self.request) - while (not event_is_set(self._shutdown_event) and + while (not self._shutdown_event.isSet() and (timeit.default_timer() - self.starttime) <= self.timeout): self.result.append(len(f.read(10240))) @@ -826,8 +748,6 @@ class HTTPDownloader(threading.Thread): f.close() except IOError: pass - except HTTP_ERRORS: - pass class HTTPUploaderData(object): @@ -873,7 +793,7 @@ class HTTPUploaderData(object): def read(self, n=10240): if ((timeit.default_timer() - self.start) <= self.timeout and - not event_is_set(self._shutdown_event)): + not self._shutdown_event.isSet()): chunk = self.data.read(n) self.total.append(len(chunk)) return chunk @@ -893,7 +813,7 @@ class HTTPUploader(threading.Thread): self.request = request self.request.data.start = self.starttime = start self.size = size - self.result = 0 + self.result = None self.timeout = timeout self.i = i @@ -911,7 +831,7 @@ class HTTPUploader(threading.Thread): request = self.request try: if ((timeit.default_timer() - self.starttime) <= self.timeout and - not event_is_set(self._shutdown_event)): + not self._shutdown_event.isSet()): try: f = self._opener(request) except TypeError: @@ -928,8 +848,6 @@ class HTTPUploader(threading.Thread): self.result = 0 except (IOError, SpeedtestUploadTimeout): self.result = sum(self.request.data.total) - except HTTP_ERRORS: - self.result = 0 class SpeedtestResults(object): @@ -1118,7 +1036,10 @@ class Speedtest(object): @property def best(self): if not self._best: - self.get_best_server() + raise SpeedtestMissingBestServer( + 'get_best_server not called or not able to determine best ' + 'server' + ) return self._best def get_config(self): @@ -1183,9 +1104,9 @@ class Speedtest(object): # times = get_attributes_by_tag_name(root, 'times') client = get_attributes_by_tag_name(root, 'client') - ignore_servers = [ - int(i) for i in server_config['ignoreids'].split(',') if i - ] + ignore_servers = list( + map(int, server_config['ignoreids'].split(',')) + ) ratio = int(upload['ratio']) upload_max = int(upload['maxchunkcount']) @@ -1313,7 +1234,7 @@ class Speedtest(object): raise SpeedtestServersError( 'Malformed speedtest.net server list: %s' % e ) - elements = etree_iter(root, 'server') + elements = root.getiterator('server') except AttributeError: try: root = DOM.parseString(serversxml) @@ -1513,12 +1434,8 @@ class Speedtest(object): printer('Best Server:\n%r' % best, debug=True) return best - def download(self, callback=do_nothing, threads=None): - """Test download speed against speedtest.net - - A ``threads`` value of ``None`` will fall back to those dictated - by the speedtest.net configuration - """ + def download(self, callback=do_nothing): + """Test download speed against speedtest.net""" urls = [] for size in self.config['sizes']['download']: @@ -1533,9 +1450,6 @@ class Speedtest(object): build_request(url, bump=i, secure=self._secure) ) - max_threads = threads or self.config['threads']['download'] - in_flight = {'threads': 0} - def producer(q, requests, request_count): for i, request in enumerate(requests): thread = HTTPDownloader( @@ -1546,26 +1460,21 @@ class Speedtest(object): opener=self._opener, shutdown_event=self._shutdown_event ) - while in_flight['threads'] >= max_threads: - timeit.time.sleep(0.001) thread.start() q.put(thread, True) - in_flight['threads'] += 1 callback(i, request_count, start=True) finished = [] def consumer(q, request_count): - _is_alive = thread_is_alive while len(finished) < request_count: thread = q.get(True) - while _is_alive(thread): - thread.join(timeout=0.001) - in_flight['threads'] -= 1 + while thread.isAlive(): + thread.join(timeout=0.1) finished.append(sum(thread.result)) callback(thread.i, request_count, end=True) - q = Queue(max_threads) + q = Queue(self.config['threads']['download']) prod_thread = threading.Thread(target=producer, args=(q, requests, request_count)) cons_thread = threading.Thread(target=consumer, @@ -1573,11 +1482,10 @@ class Speedtest(object): start = timeit.default_timer() prod_thread.start() cons_thread.start() - _is_alive = thread_is_alive - while _is_alive(prod_thread): - prod_thread.join(timeout=0.001) - while _is_alive(cons_thread): - cons_thread.join(timeout=0.001) + while prod_thread.isAlive(): + prod_thread.join(timeout=0.1) + while cons_thread.isAlive(): + cons_thread.join(timeout=0.1) stop = timeit.default_timer() self.results.bytes_received = sum(finished) @@ -1588,12 +1496,8 @@ class Speedtest(object): self.config['threads']['upload'] = 8 return self.results.download - def upload(self, callback=do_nothing, pre_allocate=True, threads=None): - """Test upload speed against speedtest.net - - A ``threads`` value of ``None`` will fall back to those dictated - by the speedtest.net configuration - """ + def upload(self, callback=do_nothing, pre_allocate=True): + """Test upload speed against speedtest.net""" sizes = [] @@ -1616,19 +1520,13 @@ class Speedtest(object): ) if pre_allocate: data.pre_allocate() - - headers = {'Content-length': size} requests.append( ( - build_request(self.best['url'], data, secure=self._secure, - headers=headers), + build_request(self.best['url'], data, secure=self._secure), size ) ) - max_threads = threads or self.config['threads']['upload'] - in_flight = {'threads': 0} - def producer(q, requests, request_count): for i, request in enumerate(requests[:request_count]): thread = HTTPUploader( @@ -1640,26 +1538,21 @@ class Speedtest(object): opener=self._opener, shutdown_event=self._shutdown_event ) - while in_flight['threads'] >= max_threads: - timeit.time.sleep(0.001) thread.start() q.put(thread, True) - in_flight['threads'] += 1 callback(i, request_count, start=True) finished = [] def consumer(q, request_count): - _is_alive = thread_is_alive while len(finished) < request_count: thread = q.get(True) - while _is_alive(thread): - thread.join(timeout=0.001) - in_flight['threads'] -= 1 + while thread.isAlive(): + thread.join(timeout=0.1) finished.append(thread.result) callback(thread.i, request_count, end=True) - q = Queue(threads or self.config['threads']['upload']) + q = Queue(self.config['threads']['upload']) prod_thread = threading.Thread(target=producer, args=(q, requests, request_count)) cons_thread = threading.Thread(target=consumer, @@ -1667,10 +1560,9 @@ class Speedtest(object): start = timeit.default_timer() prod_thread.start() cons_thread.start() - _is_alive = thread_is_alive - while _is_alive(prod_thread): + while prod_thread.isAlive(): prod_thread.join(timeout=0.1) - while _is_alive(cons_thread): + while cons_thread.isAlive(): cons_thread.join(timeout=0.1) stop = timeit.default_timer() @@ -1695,8 +1587,7 @@ def ctrl_c(shutdown_event): def version(): """Print the version""" - printer('speedtest-cli %s' % __version__) - printer('Python %s' % sys.version.replace('\n', '')) + printer(__version__) sys.exit(0) @@ -1729,10 +1620,6 @@ def parse_args(): parser.add_argument('--no-upload', dest='upload', default=True, action='store_const', const=False, help='Do not perform upload test') - parser.add_argument('--single', default=False, action='store_true', - help='Only use a single connection instead of ' - 'multiple. This simulates a typical file ' - 'transfer.') parser.add_argument('--bytes', dest='units', action='store_const', const=('byte', 8), default=('bit', 1), help='Display values in bytes instead of bits. Does ' @@ -1947,10 +1834,7 @@ def shell(): if args.download: printer('Testing download speed', quiet, end=('', '\n')[bool(debug)]) - speedtest.download( - callback=callback, - threads=(None, 1)[args.single] - ) + speedtest.download(callback=callback) printer('Download: %0.2f M%s/s' % ((results.download / 1000.0 / 1000.0) / args.units[1], args.units[0]), @@ -1961,11 +1845,7 @@ def shell(): if args.upload: printer('Testing upload speed', quiet, end=('', '\n')[bool(debug)]) - speedtest.upload( - callback=callback, - pre_allocate=args.pre_allocate, - threads=(None, 1)[args.single] - ) + speedtest.upload(callback=callback, pre_allocate=args.pre_allocate) printer('Upload: %0.2f M%s/s' % ((results.upload / 1000.0 / 1000.0) / args.units[1], args.units[0]), @@ -2003,10 +1883,7 @@ def main(): e = get_exception() # Ignore a successful exit, or argparse exit if getattr(e, 'code', 1) not in (0, 2): - msg = '%s' % e - if not msg: - msg = '%r' % e - raise SystemExit('ERROR: %s' % msg) + raise SystemExit('ERROR: %s' % e) if __name__ == '__main__':