Release 0.5.4

servefile still works with python3.6, we just no longer test this.

.github/workflows/run-tox.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [2.7, 3.6, 3.7, 3.8, 3.9]
+        python: [2.7, 3.7, 3.8, 3.9, "3.10", 3.11]
 
     steps:
       - uses: actions/checkout@v2
@@ -23,3 +23,15 @@ jobs:
         run: pip install tox
       - name: Run Tox
         run: tox -e py
+  pep8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox pep8
+        run: "tox -e pep8"

ChangeLog

@@ -1,6 +1,24 @@
 servefile changelog
 ===================
 
+2023-01-23 v0.5.4
+-----------------
+
+	0.5.4 released
+
+	* code reformatting for better maintainability
+	* upload to uploaddir instead of /tmp for large files
+	* add python3.10 / python3.11 support
+	* drop python3.6 support
+
+
+2021-11-18 v0.5.3
+-----------------
+
+	0.5.3 released
+
+	* improved test performance
+
 
 2021-09-08 v0.5.2
 -----------------

servefile.1

@@ -1,4 +1,4 @@
-.TH SERVEFILE 1 "September 2021" "servefile 0.5.2" "User Commands"
+.TH SERVEFILE 1 "January 2023" "servefile 0.5.4" "User Commands"
 
 .SH NAME
 servefile \- small HTTP-Server for temporary file transfer

servefile/servefile.py

@@ -7,11 +7,10 @@
 
 from __future__ import print_function
 
-__version__ = '0.5.2'
+__version__ = '0.5.4'
 
 import argparse
 import base64
-import cgi
 import datetime
 import io
 import mimetypes
@@ -21,7 +20,9 @@ import select
 import socket
 from subprocess import Popen, PIPE
 import sys
+import tempfile
 import time
+import warnings
 
 # fix imports for python2/python3
 try:
@@ -42,6 +43,11 @@ try:
 except ImportError:
     pass
 
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+    # scheduled for removal in python3.13, used for FieldStorage
+    import cgi
+
 
 def getDateStrNow():
     """ Get the current time formatted for HTTP header """
@@ -526,10 +532,7 @@ class DirListingHandler(FileBaseHandler):
             target_items.append((item, itemPath, stat))
 
         # Directories first, then files
-        for (tuple_list, is_dir) in (
-                (dir_items, True),
-                (file_items, False),
-                ):
+        for (tuple_list, is_dir) in ((dir_items, True), (file_items, False)):
             for (item, itemPath, stat) in tuple_list:
                 self._appendToListing(content, item, itemPath, stat, is_dir=is_dir)
 
@@ -611,7 +614,24 @@ class FilePutter(BaseHTTPServer.BaseHTTPRequestHandler):
         # create FieldStorage object for multipart parsing
         env = os.environ
         env['REQUEST_METHOD'] = "POST"
-        fstorage = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ=env)
+        targetDir = self.targetDir
+
+        class CustomFieldStorage(cgi.FieldStorage):
+
+            def make_file(self, *args, **kwargs):
+                """Overwritten to use a named file and the upload directory
+
+                Python 2.7 has an unused "binary" argument while Python 3 does
+                not have any arguments. Python 2.7 does not have a
+                self._binary_file attribute.
+                """
+                if sys.version_info.major == 2 or self._binary_file:
+                    return tempfile.NamedTemporaryFile("wb+", dir=targetDir)
+                else:
+                    return tempfile.NamedTemporaryFile(
+                        "w+", encoding=self.encoding, newline='\n', dir=targetDir)
+
+        fstorage = CustomFieldStorage(fp=self.rfile, headers=self.headers, environ=env)
         if "file" not in fstorage:
             self.sendResponse(400, "No file found in request.")
             return
@@ -621,7 +641,14 @@ class FilePutter(BaseHTTPServer.BaseHTTPRequestHandler):
             self.sendResponse(400, "Filename was empty or invalid")
             return
 
-        # write file down to disk, send a 200 afterwards
+        # put the file at the right place, send 200 afterwards
+        if getattr(fstorage["file"].file, "name", None):
+            # the sent file was large, so we can just hard link the temporary
+            # file and are done
+            os.link(fstorage["file"].file.name, destFileName)
+        else:
+            # write file to disk. it was small enough so no temporary file was
+            # created
             target = open(destFileName, "wb")
             bytesLeft = length
             while bytesLeft > 0:
@@ -849,7 +876,7 @@ class ServeFile():
             if oldLang:
                 os.environ['LC_ALL'] = oldLang
             else:
-                del(os.environ['LC_ALL'])
+                del os.environ['LC_ALL']
             if proc.wait() != 0:
                 # we couldn't find any ip address
                 proc = None

setup.py

@@ -11,7 +11,7 @@ setup(
     long_description=long_description,
     long_description_content_type='text/markdown',
     platforms='posix',
-    version='0.5.2',
+    version='0.5.4',
     license='GPLv3 or later',
     url='https://github.com/sebageek/servefile/',
     author='Sebastian Lohff',
@@ -38,10 +38,11 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Communications',
         'Topic :: Communications :: File Sharing',
         'Topic :: Internet',

tests/test_servefile.py

@@ -9,6 +9,7 @@ import sys
 import tarfile
 import time
 import urllib3
+from requests.exceptions import ConnectionError
 
 # crudly written to learn more about pytest and to have a base for refactoring
 
@@ -58,7 +59,6 @@ def run_servefile():
 
         print("running {} with args {}".format(", ".join(servefile_path), args))
         p = subprocess.Popen([sys.executable] + servefile_path + args, **kwargs)
-        time.sleep(kwargs.get('timeout', 0.3))
         instances.append(p)
 
         return p
@@ -91,7 +91,7 @@ def datadir(tmp_path):
     return _datadir
 
 
-def make_request(path='/', host='localhost', port=SERVEFILE_DEFAULT_PORT, method='get', protocol='http', timeout=5,
+def make_request(path='/', host='localhost', port=SERVEFILE_DEFAULT_PORT, method='get', protocol='http',
                  encoding='utf-8', **kwargs):
     url = '{}://{}:{}{}'.format(protocol, host, port, path)
     print('Calling {} on {} with {}'.format(method, url, kwargs))
@@ -103,7 +103,7 @@ def make_request(path='/', host='localhost', port=SERVEFILE_DEFAULT_PORT, method
     return r
 
 
-def check_download(expected_data=None, path='/', fname=None, status_code=200, **kwargs):
+def check_download(expected_data=None, path='/', fname=None, **kwargs):
     if fname is None:
         fname = os.path.basename(path)
     r = make_request(path, **kwargs)
@@ -117,6 +117,22 @@ def check_download(expected_data=None, path='/', fname=None, status_code=200, **
     return r  # for additional tests
 
 
+def _retry_while(exception, function, timeout=2):
+    now = time.time  # float seconds since epoch
+
+    def wrapped(*args, **kwargs):
+        timeout_after = now() + timeout
+        while True:
+            try:
+                return function(*args, **kwargs)
+            except exception:
+                if now() >= timeout_after:
+                    raise
+                time.sleep(0.1)
+
+    return wrapped
+
+
 def _test_version(run_servefile, standalone):
     # we expect the version on stdout (python3.4+) or stderr(python2.6-3.3)
     s = run_servefile('--version', standalone=standalone, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -130,7 +146,7 @@ def _test_version(run_servefile, standalone):
         version = s.stdout.readline().decode().strip()
 
     # hardcode version as string until servefile is a module
-    assert version == 'servefile 0.5.2'
+    assert version == 'servefile 0.5.4'
 
 
 def test_version(run_servefile):
@@ -147,7 +163,7 @@ def test_correct_headers(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(str(p))
 
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     assert r.headers.get('Content-Type') == 'application/octet-stream'
     assert r.headers.get('Content-Disposition') == 'attachment; filename="testfile"'
@@ -160,7 +176,7 @@ def test_redirect_and_download(run_servefile, datadir):
     run_servefile(str(p))
 
     # redirect
-    r = make_request(allow_redirects=False)
+    r = _retry_while(ConnectionError, make_request)(allow_redirects=False)
     assert r.status_code == 302
     assert r.headers.get('Location') == '/testfile'
 
@@ -175,7 +191,7 @@ def test_redirect_and_download_with_umlaut(run_servefile, datadir):
     run_servefile(str(p))
 
     # redirect
-    r = make_request(allow_redirects=False)
+    r = _retry_while(ConnectionError, make_request)(allow_redirects=False)
     assert r.status_code == 302
     assert r.headers.get('Location') == '/{}'.format(quote(filename))
 
@@ -190,7 +206,7 @@ def test_specify_port(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile([str(p), '-p', str(SERVEFILE_SECONDARY_PORT)])
 
-    check_download(data, fname='testfile', port=SERVEFILE_SECONDARY_PORT)
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', port=SERVEFILE_SECONDARY_PORT)
 
 
 def test_ipv4_only(run_servefile, datadir):
@@ -198,7 +214,7 @@ def test_ipv4_only(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile([str(p), '-4'])
 
-    check_download(data, fname='testfile', host='127.0.0.1')
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', host='127.0.0.1')
 
     sock = socket.socket(socket.AF_INET6)
     with pytest.raises(connrefused_exc):
@@ -211,7 +227,7 @@ def test_big_download(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(str(p))
 
-    check_download(data, fname='testfile')
+    _retry_while(ConnectionError, check_download)(data, fname='testfile')
 
 
 def test_authentication(run_servefile, datadir):
@@ -220,11 +236,11 @@ def test_authentication(run_servefile, datadir):
 
     run_servefile([str(p), '-a', 'user:password'])
     for auth in [('foo', 'bar'), ('user', 'wrong'), ('unknown', 'password')]:
-        r = make_request(auth=auth)
+        r = _retry_while(ConnectionError, make_request)(auth=auth)
         assert '401 - Unauthorized' in r.text
         assert r.status_code == 401
 
-    check_download(data, fname='testfile', auth=('user', 'password'))
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', auth=('user', 'password'))
 
 
 def test_serve_directory(run_servefile, datadir):
@@ -241,12 +257,12 @@ def test_serve_directory(run_servefile, datadir):
     # check if all files are in directory listing
     # (could be made more sophisticated with beautifulsoup)
     for path in '/', '/../':
-        r = make_request(path)
+        r = _retry_while(ConnectionError, make_request)(path)
         for k in d:
             assert quote(k) in r.text
 
     for fname, content in d['foo'].items():
-        check_download(content, '/foo/' + fname)
+        _retry_while(ConnectionError, check_download)(content, '/foo/' + fname)
 
     r = make_request('/unknown')
     assert r.status_code == 404
@@ -268,7 +284,7 @@ def test_serve_relative_directory(run_servefile, datadir):
     # check if all files are in directory listing
     # (could be made more sophisticated with beautifulsoup)
     for path in '/', '/../':
-        r = make_request(path)
+        r = _retry_while(ConnectionError, make_request)(path)
         for k in d:
             assert k in r.text
 
@@ -292,14 +308,14 @@ def test_upload(run_servefile, tmp_path):
 
     run_servefile(['-u', str(uploaddir)])
 
-    # check that servefile created the directory
-    assert uploaddir.is_dir()
-
     # check upload form present
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     assert 'multipart/form-data' in r.text
 
+    # check that servefile created the directory
+    assert uploaddir.is_dir()
+
     # upload file
     files = {'file': ('haiku.txt', data)}
     r = make_request(method='post', files=files)
@@ -329,7 +345,7 @@ def test_upload_size_limit(run_servefile, tmp_path):
 
     # upload file that is too big
     files = {'file': ('toobig', "x" * 2049)}
-    r = make_request(method='post', files=files)
+    r = _retry_while(ConnectionError, make_request)(method='post', files=files)
     assert 'Your file was too big' in r.text
     assert r.status_code == 413
     assert not (uploaddir / 'toobig').exists()
@@ -341,6 +357,20 @@ def test_upload_size_limit(run_servefile, tmp_path):
     assert r.status_code == 200
 
 
+def test_upload_large_file(run_servefile, tmp_path):
+    # small files end up in BytesIO while large files get temporary files. this
+    # test makes sure we hit the large file codepath at least once
+    uploaddir = tmp_path / 'upload'
+    run_servefile(['-u', str(uploaddir)])
+
+    data = "asdf" * 1024
+    files = {'file': ('more_data.txt', data)}
+    r = _retry_while(ConnectionError, make_request)(method='post', files=files)
+    assert r.status_code == 200
+    with open(str(uploaddir / 'more_data.txt')) as f:
+        assert f.read() == data
+
+
 def test_tar_mode(run_servefile, datadir):
     d = {
         'foo': {
@@ -354,7 +384,7 @@ def test_tar_mode(run_servefile, datadir):
     # test redirect?
 
     # test contents of tar file
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     tar = tarfile.open(fileobj=io.BytesIO(r.content))
     assert len(tar.getmembers()) == 3
@@ -370,7 +400,7 @@ def test_tar_compression(run_servefile, datadir):
     p = datadir(d)
     run_servefile(['-c', 'gzip', '-t', str(p / 'foo')])
 
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     tar = tarfile.open(fileobj=io.BytesIO(r.content), mode='r:gz')
     assert len(tar.getmembers()) == 1
@@ -380,7 +410,6 @@ def test_https(run_servefile, datadir):
     data = "NOOT NOOT"
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(['--ssl', str(p)])
-    time.sleep(0.2)  # time for generating ssl certificates
 
     # fingerprint = None
     # while not fingerprint:
@@ -395,7 +424,7 @@ def test_https(run_servefile, datadir):
 
     # assert fingerprint
     urllib3.disable_warnings()
-    check_download(data, protocol='https', verify=False)
+    _retry_while(ConnectionError, check_download)(data, protocol='https', verify=False)
 
 
 def test_https_big_download(run_servefile, datadir):
@@ -403,10 +432,9 @@ def test_https_big_download(run_servefile, datadir):
     data = "x" * (10 * 1024 ** 2)
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(['--ssl', str(p)])
-    time.sleep(0.2)  # time for generating ssl certificates
 
     urllib3.disable_warnings()
-    check_download(data, protocol='https', verify=False)
+    _retry_while(ConnectionError, check_download)(data, protocol='https', verify=False)
 
 
 def test_abort_download(run_servefile, datadir):
@@ -419,7 +447,7 @@ def test_abort_download(run_servefile, datadir):
     # provoke a connection abort
     # hopefully the buffers will not fill up with all of the 10mb
     sock = socket.socket(socket.AF_INET)
-    sock.connect(("localhost", SERVEFILE_DEFAULT_PORT))
+    _retry_while(connrefused_exc, sock.connect)(("localhost", SERVEFILE_DEFAULT_PORT))
     sock.send(b"GET /testfile HTTP/1.0\n\n")
     resp = sock.recv(100)
     assert resp != b''

tox.ini

@@ -1,9 +1,19 @@
 [tox]
-envlist = py27,py36,py37,py38,py39
+envlist = py27,py37,py38,py39,py310,py311,pep8
 
 [testenv]
 deps =
         pathlib2; python_version<"3"
         pytest
         requests
-commands = pytest --tb=short {posargs}
+        flake8
+commands = pytest -v --tb=short {posargs}
+
+[testenv:pep8]
+commands = flake8 servefile/ {posargs}
+
+[flake8]
+show-source = True
+max-line-length = 120
+ignore = E123,E125,E241,E402,E741,W503,W504,H301
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build