Release 0.5.4

servefile still works with python3.6, we just no longer test this.

.github/workflows/run-tox.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [2.7, 3.6, 3.7, 3.8, 3.9]
+        python: [2.7, 3.7, 3.8, 3.9, "3.10", 3.11]
 
     steps:
       - uses: actions/checkout@v2
@@ -23,3 +23,15 @@ jobs:
         run: pip install tox
       - name: Run Tox
         run: tox -e py
+  pep8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox pep8
+        run: "tox -e pep8"

ChangeLog

@@ -1,9 +1,29 @@
 servefile changelog
 ===================
 
+2023-01-23 v0.5.4
+-----------------
+
+	0.5.4 released
+
+	* code reformatting for better maintainability
+	* upload to uploaddir instead of /tmp for large files
+	* add python3.10 / python3.11 support
+	* drop python3.6 support
+
+
+2021-11-18 v0.5.3
+-----------------
+
+	0.5.3 released
+
+	* improved test performance
+
+
+2021-09-08 v0.5.2
+-----------------
 
-Unreleased
-----------
+	0.5.2 released
 
 	* fixed bug where exception was shown on transmission abort with python3
 	* fixed wrong/outdated pyopenssl package names

servefile.1

@@ -1,4 +1,4 @@
-.TH SERVEFILE 1 "September 2020" "servefile 0.5.1" "User Commands"
+.TH SERVEFILE 1 "January 2023" "servefile 0.5.4" "User Commands"
 
 .SH NAME
 servefile \- small HTTP-Server for temporary file transfer

setup.py

@@ -11,7 +11,7 @@ setup(
     long_description=long_description,
     long_description_content_type='text/markdown',
     platforms='posix',
-    version='0.5.1',
+    version='0.5.4',
     license='GPLv3 or later',
     url='https://github.com/sebageek/servefile/',
     author='Sebastian Lohff',
@@ -38,10 +38,11 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Communications',
         'Topic :: Communications :: File Sharing',
         'Topic :: Internet',

tests/test_servefile.py

@@ -9,6 +9,7 @@ import sys
 import tarfile
 import time
 import urllib3
+from requests.exceptions import ConnectionError
 
 # crudly written to learn more about pytest and to have a base for refactoring
 
@@ -58,7 +59,6 @@ def run_servefile():
 
         print("running {} with args {}".format(", ".join(servefile_path), args))
         p = subprocess.Popen([sys.executable] + servefile_path + args, **kwargs)
-        time.sleep(kwargs.get('timeout', 0.3))
         instances.append(p)
 
         return p
@@ -91,16 +91,19 @@ def datadir(tmp_path):
     return _datadir
 
 
-def make_request(path='/', host='localhost', port=SERVEFILE_DEFAULT_PORT, method='get', protocol='http', timeout=5,
-                 **kwargs):
+def make_request(path='/', host='localhost', port=SERVEFILE_DEFAULT_PORT, method='get', protocol='http',
+                 encoding='utf-8', **kwargs):
     url = '{}://{}:{}{}'.format(protocol, host, port, path)
     print('Calling {} on {} with {}'.format(method, url, kwargs))
     r = getattr(requests, method)(url, **kwargs)
 
+    if r.encoding is None and encoding:
+        r.encoding = encoding
+
     return r
 
 
-def check_download(expected_data=None, path='/', fname=None, status_code=200, **kwargs):
+def check_download(expected_data=None, path='/', fname=None, **kwargs):
     if fname is None:
         fname = os.path.basename(path)
     r = make_request(path, **kwargs)
@@ -114,6 +117,22 @@ def check_download(expected_data=None, path='/', fname=None, status_code=200, **
     return r  # for additional tests
 
 
+def _retry_while(exception, function, timeout=2):
+    now = time.time  # float seconds since epoch
+
+    def wrapped(*args, **kwargs):
+        timeout_after = now() + timeout
+        while True:
+            try:
+                return function(*args, **kwargs)
+            except exception:
+                if now() >= timeout_after:
+                    raise
+                time.sleep(0.1)
+
+    return wrapped
+
+
 def _test_version(run_servefile, standalone):
     # we expect the version on stdout (python3.4+) or stderr(python2.6-3.3)
     s = run_servefile('--version', standalone=standalone, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -127,7 +146,7 @@ def _test_version(run_servefile, standalone):
         version = s.stdout.readline().decode().strip()
 
     # hardcode version as string until servefile is a module
-    assert version == 'servefile 0.5.1'
+    assert version == 'servefile 0.5.4'
 
 
 def test_version(run_servefile):
@@ -144,7 +163,7 @@ def test_correct_headers(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(str(p))
 
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     assert r.headers.get('Content-Type') == 'application/octet-stream'
     assert r.headers.get('Content-Disposition') == 'attachment; filename="testfile"'
@@ -157,7 +176,7 @@ def test_redirect_and_download(run_servefile, datadir):
     run_servefile(str(p))
 
     # redirect
-    r = make_request(allow_redirects=False)
+    r = _retry_while(ConnectionError, make_request)(allow_redirects=False)
     assert r.status_code == 302
     assert r.headers.get('Location') == '/testfile'
 
@@ -172,7 +191,7 @@ def test_redirect_and_download_with_umlaut(run_servefile, datadir):
     run_servefile(str(p))
 
     # redirect
-    r = make_request(allow_redirects=False)
+    r = _retry_while(ConnectionError, make_request)(allow_redirects=False)
     assert r.status_code == 302
     assert r.headers.get('Location') == '/{}'.format(quote(filename))
 
@@ -187,7 +206,7 @@ def test_specify_port(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile([str(p), '-p', str(SERVEFILE_SECONDARY_PORT)])
 
-    check_download(data, fname='testfile', port=SERVEFILE_SECONDARY_PORT)
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', port=SERVEFILE_SECONDARY_PORT)
 
 
 def test_ipv4_only(run_servefile, datadir):
@@ -195,7 +214,7 @@ def test_ipv4_only(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile([str(p), '-4'])
 
-    check_download(data, fname='testfile', host='127.0.0.1')
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', host='127.0.0.1')
 
     sock = socket.socket(socket.AF_INET6)
     with pytest.raises(connrefused_exc):
@@ -208,7 +227,7 @@ def test_big_download(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(str(p))
 
-    check_download(data, fname='testfile')
+    _retry_while(ConnectionError, check_download)(data, fname='testfile')
 
 
 def test_authentication(run_servefile, datadir):
@@ -217,11 +236,11 @@ def test_authentication(run_servefile, datadir):
 
     run_servefile([str(p), '-a', 'user:password'])
     for auth in [('foo', 'bar'), ('user', 'wrong'), ('unknown', 'password')]:
-        r = make_request(auth=auth)
+        r = _retry_while(ConnectionError, make_request)(auth=auth)
         assert '401 - Unauthorized' in r.text
         assert r.status_code == 401
 
-    check_download(data, fname='testfile', auth=('user', 'password'))
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', auth=('user', 'password'))
 
 
 def test_serve_directory(run_servefile, datadir):
@@ -238,12 +257,12 @@ def test_serve_directory(run_servefile, datadir):
     # check if all files are in directory listing
     # (could be made more sophisticated with beautifulsoup)
     for path in '/', '/../':
-        r = make_request(path)
+        r = _retry_while(ConnectionError, make_request)(path)
         for k in d:
             assert quote(k) in r.text
 
     for fname, content in d['foo'].items():
-        check_download(content, '/foo/' + fname)
+        _retry_while(ConnectionError, check_download)(content, '/foo/' + fname)
 
     r = make_request('/unknown')
     assert r.status_code == 404
@@ -265,7 +284,7 @@ def test_serve_relative_directory(run_servefile, datadir):
     # check if all files are in directory listing
     # (could be made more sophisticated with beautifulsoup)
     for path in '/', '/../':
-        r = make_request(path)
+        r = _retry_while(ConnectionError, make_request)(path)
         for k in d:
             assert k in r.text
 
@@ -289,14 +308,14 @@ def test_upload(run_servefile, tmp_path):
 
     run_servefile(['-u', str(uploaddir)])
 
-    # check that servefile created the directory
-    assert uploaddir.is_dir()
-
     # check upload form present
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     assert 'multipart/form-data' in r.text
 
+    # check that servefile created the directory
+    assert uploaddir.is_dir()
+
     # upload file
     files = {'file': ('haiku.txt', data)}
     r = make_request(method='post', files=files)
@@ -326,7 +345,7 @@ def test_upload_size_limit(run_servefile, tmp_path):
 
     # upload file that is too big
     files = {'file': ('toobig', "x" * 2049)}
-    r = make_request(method='post', files=files)
+    r = _retry_while(ConnectionError, make_request)(method='post', files=files)
     assert 'Your file was too big' in r.text
     assert r.status_code == 413
     assert not (uploaddir / 'toobig').exists()
@@ -338,6 +357,20 @@ def test_upload_size_limit(run_servefile, tmp_path):
     assert r.status_code == 200
 
 
+def test_upload_large_file(run_servefile, tmp_path):
+    # small files end up in BytesIO while large files get temporary files. this
+    # test makes sure we hit the large file codepath at least once
+    uploaddir = tmp_path / 'upload'
+    run_servefile(['-u', str(uploaddir)])
+
+    data = "asdf" * 1024
+    files = {'file': ('more_data.txt', data)}
+    r = _retry_while(ConnectionError, make_request)(method='post', files=files)
+    assert r.status_code == 200
+    with open(str(uploaddir / 'more_data.txt')) as f:
+        assert f.read() == data
+
+
 def test_tar_mode(run_servefile, datadir):
     d = {
         'foo': {
@@ -351,7 +384,7 @@ def test_tar_mode(run_servefile, datadir):
     # test redirect?
 
     # test contents of tar file
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     tar = tarfile.open(fileobj=io.BytesIO(r.content))
     assert len(tar.getmembers()) == 3
@@ -367,7 +400,7 @@ def test_tar_compression(run_servefile, datadir):
     p = datadir(d)
     run_servefile(['-c', 'gzip', '-t', str(p / 'foo')])
 
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     tar = tarfile.open(fileobj=io.BytesIO(r.content), mode='r:gz')
     assert len(tar.getmembers()) == 1
@@ -377,7 +410,6 @@ def test_https(run_servefile, datadir):
     data = "NOOT NOOT"
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(['--ssl', str(p)])
-    time.sleep(0.2)  # time for generating ssl certificates
 
     # fingerprint = None
     # while not fingerprint:
@@ -392,7 +424,7 @@ def test_https(run_servefile, datadir):
 
     # assert fingerprint
     urllib3.disable_warnings()
-    check_download(data, protocol='https', verify=False)
+    _retry_while(ConnectionError, check_download)(data, protocol='https', verify=False)
 
 
 def test_https_big_download(run_servefile, datadir):
@@ -400,10 +432,9 @@ def test_https_big_download(run_servefile, datadir):
     data = "x" * (10 * 1024 ** 2)
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(['--ssl', str(p)])
-    time.sleep(0.2)  # time for generating ssl certificates
 
     urllib3.disable_warnings()
-    check_download(data, protocol='https', verify=False)
+    _retry_while(ConnectionError, check_download)(data, protocol='https', verify=False)
 
 
 def test_abort_download(run_servefile, datadir):
@@ -416,7 +447,7 @@ def test_abort_download(run_servefile, datadir):
     # provoke a connection abort
     # hopefully the buffers will not fill up with all of the 10mb
     sock = socket.socket(socket.AF_INET)
-    sock.connect(("localhost", SERVEFILE_DEFAULT_PORT))
+    _retry_while(connrefused_exc, sock.connect)(("localhost", SERVEFILE_DEFAULT_PORT))
     sock.send(b"GET /testfile HTTP/1.0\n\n")
     resp = sock.recv(100)
     assert resp != b''

tox.ini

@@ -1,9 +1,19 @@
 [tox]
-envlist = py27,py36,py37,py38,py39
+envlist = py27,py37,py38,py39,py310,py311,pep8
 
 [testenv]
 deps =
         pathlib2; python_version<"3"
         pytest
         requests
-commands = pytest --tb=short {posargs}
+        flake8
+commands = pytest -v --tb=short {posargs}
+
+[testenv:pep8]
+commands = flake8 servefile/ {posargs}
+
+[flake8]
+show-source = True
+max-line-length = 120
+ignore = E123,E125,E241,E402,E741,W503,W504,H301
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build