Allow ports for tests to be specified via env

SERVEFILE_DEFAULT_PORT and SERVEFILE_SECONDARY_PORT can be used to
specify ports used in the servefile tests. This can be useful if the
default port 8080 and the secondary port 8081 (for the -p test) are
already in use. To allow automatic choosing of a free port 0 can be
specified to tell the test code to automatically select a free port.

.github/workflows/run-tox.yml

@@ -1,37 +0,0 @@
-name: Run Tox
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python: [2.7, 3.7, 3.8, 3.9, "3.10", 3.11]
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Install Tox
-        run: pip install tox
-      - name: Run Tox
-        run: tox -e py
-  pep8:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - name: Install Tox
-        run: pip install tox
-      - name: Run Tox pep8
-        run: "tox -e pep8"

ChangeLog

@@ -1,41 +1,15 @@
 servefile changelog
 ===================
 
-2023-01-23 v0.5.4
------------------
 
-	0.5.4 released
-
-	* code reformatting for better maintainability
-	* upload to uploaddir instead of /tmp for large files
-	* add python3.10 / python3.11 support
-	* drop python3.6 support
-
-
-2021-11-18 v0.5.3
------------------
-
-	0.5.3 released
-
-	* improved test performance
-
-
-2021-09-08 v0.5.2
------------------
-
-	0.5.2 released
+Unreleased
+----------
 
 	* fixed bug where exception was shown on transmission abort with python3
 	* fixed wrong/outdated pyopenssl package names
-	* tests are now using a free non-default port to avoid clashes; if
-	  wished the ports can be set from outside by specifying the
+	* ports for running servefile tests can now be specified via the
 	  environment variables SERVEFILE_DEFAULT_PORT and
-	  SERVEFILE_SECONDARY_PORT
-	* fixed broken redirect when filename contained umlauts or other characters
-	  that should have been quoted
-	* fixed broken special char handling in directory listing for python2
-	* drop python3.5 support
-	* fixed PUT uploads with python3 and documented PUT-uploads with curl
+	  SERVEFILE_SECONDARY_PORT, use 0 for automatic selection
 
 
 2020-10-30 v0.5.1

servefile.1

@@ -1,4 +1,4 @@
-.TH SERVEFILE 1 "January 2023" "servefile 0.5.4" "User Commands"
+.TH SERVEFILE 1 "September 2020" "servefile 0.5.1" "User Commands"
 
 .SH NAME
 servefile \- small HTTP-Server for temporary file transfer

setup.py

@@ -11,7 +11,7 @@ setup(
     long_description=long_description,
     long_description_content_type='text/markdown',
     platforms='posix',
-    version='0.5.4',
+    version='0.5.1',
     license='GPLv3 or later',
     url='https://github.com/sebageek/servefile/',
     author='Sebastian Lohff',
@@ -38,11 +38,10 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
         'Topic :: Communications',
         'Topic :: Communications :: File Sharing',
         'Topic :: Internet',

tests/test_servefile.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import io
 import os
 import pytest
@@ -9,18 +8,15 @@ import sys
 import tarfile
 import time
 import urllib3
-from requests.exceptions import ConnectionError
 
 # crudly written to learn more about pytest and to have a base for refactoring
 
 
 if sys.version_info.major >= 3:
     from pathlib import Path
-    from urllib.parse import quote
     connrefused_exc = ConnectionRefusedError
 else:
     from pathlib2 import Path
-    from urllib import quote
     connrefused_exc = socket.error
 
 
@@ -35,8 +31,8 @@ def _get_port_from_env(var_name, default):
     return port
 
 
-SERVEFILE_DEFAULT_PORT = _get_port_from_env('SERVEFILE_DEFAULT_PORT', 0)
-SERVEFILE_SECONDARY_PORT = _get_port_from_env('SERVEFILE_SECONDARY_PORT', 0)
+SERVEFILE_DEFAULT_PORT = _get_port_from_env('SERVEFILE_DEFAULT_PORT', 8080)
+SERVEFILE_SECONDARY_PORT = _get_port_from_env('SERVEFILE_SECONDARY_PORT', 8081)
 
 
 @pytest.fixture
@@ -54,11 +50,12 @@ def run_servefile():
             servefile_path = ['-m', 'servefile']
 
         # use non-default default port, if one is given via env (and none via args)
-        if '-p' not in args and '--port' not in args:
+        if '-p' not in args and SERVEFILE_DEFAULT_PORT != 8080:
             args.extend(['-p', str(SERVEFILE_DEFAULT_PORT)])
 
         print("running {} with args {}".format(", ".join(servefile_path), args))
         p = subprocess.Popen([sys.executable] + servefile_path + args, **kwargs)
+        time.sleep(kwargs.get('timeout', 0.3))
         instances.append(p)
 
         return p
@@ -84,26 +81,23 @@ def datadir(tmp_path):
                 _datadir(v, new_path)
             else:
                 if hasattr(v, 'decode'):
-                    v = v.decode('utf-8')  # python2 compability
+                    v = v.decode()  # python2 compability
                 (path / k).write_text(v)
 
         return path
     return _datadir
 
 
-def make_request(path='/', host='localhost', port=SERVEFILE_DEFAULT_PORT, method='get', protocol='http',
-                 encoding='utf-8', **kwargs):
+def make_request(path='/', host='localhost', port=SERVEFILE_DEFAULT_PORT, method='get', protocol='http', timeout=5,
+                 **kwargs):
     url = '{}://{}:{}{}'.format(protocol, host, port, path)
     print('Calling {} on {} with {}'.format(method, url, kwargs))
     r = getattr(requests, method)(url, **kwargs)
 
-    if r.encoding is None and encoding:
-        r.encoding = encoding
-
     return r
 
 
-def check_download(expected_data=None, path='/', fname=None, **kwargs):
+def check_download(expected_data=None, path='/', fname=None, status_code=200, **kwargs):
     if fname is None:
         fname = os.path.basename(path)
     r = make_request(path, **kwargs)
@@ -117,22 +111,6 @@ def check_download(expected_data=None, path='/', fname=None, **kwargs):
     return r  # for additional tests
 
 
-def _retry_while(exception, function, timeout=2):
-    now = time.time  # float seconds since epoch
-
-    def wrapped(*args, **kwargs):
-        timeout_after = now() + timeout
-        while True:
-            try:
-                return function(*args, **kwargs)
-            except exception:
-                if now() >= timeout_after:
-                    raise
-                time.sleep(0.1)
-
-    return wrapped
-
-
 def _test_version(run_servefile, standalone):
     # we expect the version on stdout (python3.4+) or stderr(python2.6-3.3)
     s = run_servefile('--version', standalone=standalone, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -146,7 +124,7 @@ def _test_version(run_servefile, standalone):
         version = s.stdout.readline().decode().strip()
 
     # hardcode version as string until servefile is a module
-    assert version == 'servefile 0.5.4'
+    assert version == 'servefile 0.5.1'
 
 
 def test_version(run_servefile):
@@ -163,7 +141,7 @@ def test_correct_headers(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(str(p))
 
-    r = _retry_while(ConnectionError, make_request)()
+    r = make_request()
     assert r.status_code == 200
     assert r.headers.get('Content-Type') == 'application/octet-stream'
     assert r.headers.get('Content-Disposition') == 'attachment; filename="testfile"'
@@ -176,7 +154,7 @@ def test_redirect_and_download(run_servefile, datadir):
     run_servefile(str(p))
 
     # redirect
-    r = _retry_while(ConnectionError, make_request)(allow_redirects=False)
+    r = make_request(allow_redirects=False)
     assert r.status_code == 302
     assert r.headers.get('Location') == '/testfile'
 
@@ -184,29 +162,12 @@ def test_redirect_and_download(run_servefile, datadir):
     check_download(data, fname='testfile')
 
 
-def test_redirect_and_download_with_umlaut(run_servefile, datadir):
-    data = "NÖÖT NÖÖT"
-    filename = "tästføile"
-    p = datadir({filename: data}) / filename
-    run_servefile(str(p))
-
-    # redirect
-    r = _retry_while(ConnectionError, make_request)(allow_redirects=False)
-    assert r.status_code == 302
-    assert r.headers.get('Location') == '/{}'.format(quote(filename))
-
-    # normal download
-    if sys.version_info.major < 3:
-        data = unicode(data, 'utf-8')
-    check_download(data, fname=filename)
-
-
 def test_specify_port(run_servefile, datadir):
     data = "NOOT NOOT"
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile([str(p), '-p', str(SERVEFILE_SECONDARY_PORT)])
 
-    _retry_while(ConnectionError, check_download)(data, fname='testfile', port=SERVEFILE_SECONDARY_PORT)
+    check_download(data, fname='testfile', port=SERVEFILE_SECONDARY_PORT)
 
 
 def test_ipv4_only(run_servefile, datadir):
@@ -214,7 +175,7 @@ def test_ipv4_only(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile([str(p), '-4'])
 
-    _retry_while(ConnectionError, check_download)(data, fname='testfile', host='127.0.0.1')
+    check_download(data, fname='testfile', host='127.0.0.1')
 
     sock = socket.socket(socket.AF_INET6)
     with pytest.raises(connrefused_exc):
@@ -227,7 +188,7 @@ def test_big_download(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(str(p))
 
-    _retry_while(ConnectionError, check_download)(data, fname='testfile')
+    check_download(data, fname='testfile')
 
 
 def test_authentication(run_servefile, datadir):
@@ -236,11 +197,11 @@ def test_authentication(run_servefile, datadir):
 
     run_servefile([str(p), '-a', 'user:password'])
     for auth in [('foo', 'bar'), ('user', 'wrong'), ('unknown', 'password')]:
-        r = _retry_while(ConnectionError, make_request)(auth=auth)
+        r = make_request(auth=auth)
         assert '401 - Unauthorized' in r.text
         assert r.status_code == 401
 
-    _retry_while(ConnectionError, check_download)(data, fname='testfile', auth=('user', 'password'))
+    check_download(data, fname='testfile', auth=('user', 'password'))
 
 
 def test_serve_directory(run_servefile, datadir):
@@ -249,7 +210,6 @@ def test_serve_directory(run_servefile, datadir):
         'bar': {'thisisaverylongfilenamefortestingthatthisstillworksproperly': 'jup!'},
         'noot': 'still data in here',
         'bigfile': 'x' * (10 * 1024 ** 2),
-        'möwe': 'KRAKRAKRAKA',
     }
     p = datadir(d)
     run_servefile([str(p), '-l'])
@@ -257,12 +217,12 @@ def test_serve_directory(run_servefile, datadir):
     # check if all files are in directory listing
     # (could be made more sophisticated with beautifulsoup)
     for path in '/', '/../':
-        r = _retry_while(ConnectionError, make_request)(path)
+        r = make_request(path)
         for k in d:
-            assert quote(k) in r.text
+            assert k in r.text
 
     for fname, content in d['foo'].items():
-        _retry_while(ConnectionError, check_download)(content, '/foo/' + fname)
+        check_download(content, '/foo/' + fname)
 
     r = make_request('/unknown')
     assert r.status_code == 404
@@ -284,7 +244,7 @@ def test_serve_relative_directory(run_servefile, datadir):
     # check if all files are in directory listing
     # (could be made more sophisticated with beautifulsoup)
     for path in '/', '/../':
-        r = _retry_while(ConnectionError, make_request)(path)
+        r = make_request(path)
         for k in d:
             assert k in r.text
 
@@ -308,14 +268,14 @@ def test_upload(run_servefile, tmp_path):
 
     run_servefile(['-u', str(uploaddir)])
 
-    # check upload form present
-    r = _retry_while(ConnectionError, make_request)()
-    assert r.status_code == 200
-    assert 'multipart/form-data' in r.text
-
     # check that servefile created the directory
     assert uploaddir.is_dir()
 
+    # check upload form present
+    r = make_request()
+    assert r.status_code == 200
+    assert 'multipart/form-data' in r.text
+
     # upload file
     files = {'file': ('haiku.txt', data)}
     r = make_request(method='post', files=files)
@@ -331,13 +291,6 @@ def test_upload(run_servefile, tmp_path):
     with open(str(uploaddir / 'haiku.txt(1)')) as f:
         assert f.read() == data
 
-    # upload file using PUT
-    r = make_request("/haiku.txt", method='put', data=data)
-    assert r.status_code == 201
-    assert 'OK!' in r.text
-    with open(str(uploaddir / 'haiku.txt(2)')) as f:
-        assert f.read() == data
-
 
 def test_upload_size_limit(run_servefile, tmp_path):
     uploaddir = tmp_path / 'upload'
@@ -345,7 +298,7 @@ def test_upload_size_limit(run_servefile, tmp_path):
 
     # upload file that is too big
     files = {'file': ('toobig', "x" * 2049)}
-    r = _retry_while(ConnectionError, make_request)(method='post', files=files)
+    r = make_request(method='post', files=files)
     assert 'Your file was too big' in r.text
     assert r.status_code == 413
     assert not (uploaddir / 'toobig').exists()
@@ -357,20 +310,6 @@ def test_upload_size_limit(run_servefile, tmp_path):
     assert r.status_code == 200
 
 
-def test_upload_large_file(run_servefile, tmp_path):
-    # small files end up in BytesIO while large files get temporary files. this
-    # test makes sure we hit the large file codepath at least once
-    uploaddir = tmp_path / 'upload'
-    run_servefile(['-u', str(uploaddir)])
-
-    data = "asdf" * 1024
-    files = {'file': ('more_data.txt', data)}
-    r = _retry_while(ConnectionError, make_request)(method='post', files=files)
-    assert r.status_code == 200
-    with open(str(uploaddir / 'more_data.txt')) as f:
-        assert f.read() == data
-
-
 def test_tar_mode(run_servefile, datadir):
     d = {
         'foo': {
@@ -384,7 +323,7 @@ def test_tar_mode(run_servefile, datadir):
     # test redirect?
 
     # test contents of tar file
-    r = _retry_while(ConnectionError, make_request)()
+    r = make_request()
     assert r.status_code == 200
     tar = tarfile.open(fileobj=io.BytesIO(r.content))
     assert len(tar.getmembers()) == 3
@@ -400,7 +339,7 @@ def test_tar_compression(run_servefile, datadir):
     p = datadir(d)
     run_servefile(['-c', 'gzip', '-t', str(p / 'foo')])
 
-    r = _retry_while(ConnectionError, make_request)()
+    r = make_request()
     assert r.status_code == 200
     tar = tarfile.open(fileobj=io.BytesIO(r.content), mode='r:gz')
     assert len(tar.getmembers()) == 1
@@ -410,6 +349,7 @@ def test_https(run_servefile, datadir):
     data = "NOOT NOOT"
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(['--ssl', str(p)])
+    time.sleep(0.2)  # time for generating ssl certificates
 
     # fingerprint = None
     # while not fingerprint:
@@ -424,7 +364,7 @@ def test_https(run_servefile, datadir):
 
     # assert fingerprint
     urllib3.disable_warnings()
-    _retry_while(ConnectionError, check_download)(data, protocol='https', verify=False)
+    check_download(data, protocol='https', verify=False)
 
 
 def test_https_big_download(run_servefile, datadir):
@@ -432,9 +372,10 @@ def test_https_big_download(run_servefile, datadir):
     data = "x" * (10 * 1024 ** 2)
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(['--ssl', str(p)])
+    time.sleep(0.2)  # time for generating ssl certificates
 
     urllib3.disable_warnings()
-    _retry_while(ConnectionError, check_download)(data, protocol='https', verify=False)
+    check_download(data, protocol='https', verify=False)
 
 
 def test_abort_download(run_servefile, datadir):
@@ -447,7 +388,7 @@ def test_abort_download(run_servefile, datadir):
     # provoke a connection abort
     # hopefully the buffers will not fill up with all of the 10mb
     sock = socket.socket(socket.AF_INET)
-    _retry_while(connrefused_exc, sock.connect)(("localhost", SERVEFILE_DEFAULT_PORT))
+    sock.connect(("localhost", SERVEFILE_DEFAULT_PORT))
     sock.send(b"GET /testfile HTTP/1.0\n\n")
     resp = sock.recv(100)
     assert resp != b''

tox.ini

@@ -1,19 +1,9 @@
 [tox]
-envlist = py27,py37,py38,py39,py310,py311,pep8
+envlist = py27,py36,py37,py38
 
 [testenv]
 deps =
         pathlib2; python_version<"3"
         pytest
         requests
-        flake8
-commands = pytest -v --tb=short {posargs}
-
-[testenv:pep8]
-commands = flake8 servefile/ {posargs}
-
-[flake8]
-show-source = True
-max-line-length = 120
-ignore = E123,E125,E241,E402,E741,W503,W504,H301
-exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build
+commands = pytest --tb=short {posargs}