Commit 547a9f97 authored by Per Cederqvist's avatar Per Cederqvist

Display a progress bar while converting a wav file to mp3.

Automatically insert a download link for the converted file once the
conversions completes.

This uses jquery on the client side.  Two new API methods are
provided:

    <...>.mp3/convert_start
    <...>.mp3/convert_cont/<progress>/<total>

The first starts the conversion and returns an AJAX dict with the
following items:

    "status": "enqueued"
    "progress": 0
    "workload": X

X is the number of megabytes to convert (including both the current
wav file and any previously queued wav files that will be converted
before the current one).

If somebody else converted the file before the request was made, it
may also return:

    "status": "done"
    "size": human-readable size of the mp3 file
    "link": download link

The convert_cont API waits until the progress information is
significantly different than what the client says it last saw, but
always waits at least 0.2 seconds.  Then it returns a JSON-encoded
dictionary:

    "status": "progress".
    "progress": Y
    "workload"": X

X is as in the "enqueued" mapping above, and Y is the progress so far.
Once Y == X, the conversion is complete.  If that has happened,
convert_cont will instead return a mapping with status "done" (see
above).

Redesigned how wav2mp3d reports and keeps track of progress.  The
wav2mp3c demo client is not updated, so it probably no longer works.
parent 1c0ad24a
......@@ -25,4 +25,7 @@ install: all
mv -f $(WSGIDIR)/mp3wavcfg.py.tmp $(WSGIDIR)/mp3wavcfg.py
cp mp3wavweb.py $(WSGIDIR)/
cp wav2mp3d.py $(WSGIDIR)/
mkdir -p $(WSGIDIR)/mp3wavweb/js
cp mp3wavweb/js/jquery-2.1.0.min.js $(WSGIDIR)/mp3wavweb/js/
cp mp3wavweb/js/mp3wavweb.js $(WSGIDIR)/mp3wavweb/js/
service apache2 reload
......@@ -2,6 +2,7 @@ import os
import stat
import time
import io
import json
import urllib.parse
import sys
import multiprocessing.connection
......@@ -14,6 +15,9 @@ import mp3wavcfg
# to ensure we never serve partially written files.
MIN_AGE = 122
# How many jumps will the progress bar at most do?
FRACTIONS = 20
def quote(s):
return urllib.parse.quote(s.encode("utf-8"))
......@@ -60,6 +64,11 @@ class NotFound(Response):
def __init__(self):
super().__init__("404 Not Found", [])
class JSONResponse(Response):
def __init__(self, data):
super().__init__("200 OK", [("Content-Type", "application/json")],
json.dumps(data))
def application(environ, start_response):
try:
return process_request(environ, start_response)
......@@ -141,13 +150,75 @@ def convert(environ, start_response, ph):
c.send(("encode", ph.rel_base()))
while True:
msg = c.recv()
if msg[0] != "progress":
if msg[0] not in ["progress", "enqueued"]:
break
raise MovedPermanently("..")
def convert_start(environ, start_response, ph):
if len(ph.op_args()) > 0:
raise NotFound()
if environ["REQUEST_METHOD"] != "POST":
raise MethodNotAllowed('POST')
c = get_daemon_connection()
c.send(("encode", ph.rel_base()))
msg = c.recv()
if msg[0] == "enqueued":
raise JSONResponse({"status":"enqueued",
"progress":0,
"workload":msg[2]})
elif msg[0] == "enofile":
raise NotFound()
elif msg[0] == "already-there":
raise JSONResponse({"status": "done",
"size": sizeof_fmt(ph.real_stat().st_size),
"link": quote(ph.basename() + ".mp3")})
else:
raise InternalError("bad response %s to encode request" % (msg[0], ))
def conversion_progress(environ, start_response, ph):
if len(ph.op_args()) != 2:
raise NotFound()
seen_progress = int(ph.op_args()[0])
original_workload = int(ph.op_args()[1])
seen_fractions = (FRACTIONS * seen_progress) // original_workload
if environ["REQUEST_METHOD"] != "POST":
raise MethodNotAllowed('POST')
time.sleep(0.2)
c = get_daemon_connection()
c.send(("subscribe", ph.rel_base()))
while True:
try:
msg = c.recv()
except EOFError:
raise InternalError("wav2mp3d died\n")
if msg[0] == "done":
raise JSONResponse({"status": "done",
"size": sizeof_fmt(ph.real_stat().st_size),
"link": quote(ph.basename() + ".mp3")})
elif msg[0] == "progress":
fn = msg[1]
progress = msg[2]
workload = msg[3]
adjusted_progress = progress + (original_workload - workload)
adjusted_fractions = (FRACTIONS * adjusted_progress) // original_workload
if adjusted_fractions != seen_fractions:
raise JSONResponse({"status": "progress",
"progress": adjusted_progress,
"workload": original_workload})
else:
raise InternalError("Unexpected response %s\n" % (msg[0],))
MP3_OPS = {
'convert': convert,
'convert_start': convert_start,
'convert_cont': conversion_progress,
}
class PathHandler(object):
......@@ -159,11 +230,14 @@ class PathHandler(object):
(self.__dirparts, self.__basename,
self.__extension, self.__ops) = self.__split_path(self.__parts)
def basename(self):
return self.__basename
def __split_path(self, parts):
pos = len(parts) - 1
while pos >= 0:
fn, ext = os.path.splitext(parts[pos])
if ext in [".mp3", ".wav"]:
if ext in [".mp3", ".wav", ".js"]:
return parts[:pos], fn, ext, parts[pos+1:]
pos -= 1
if len(parts) == 0:
......@@ -177,7 +251,11 @@ class PathHandler(object):
def real_stat_and_file(self, ext_override=None):
path_parts = self.path_parts(ext_override)
real_file = os.path.join(mp3wavcfg.BASE, *path_parts)
if self.extension(ext_override) == ".js":
real_file = os.path.join(os.path.dirname(__file__), *path_parts)
else:
real_file = os.path.join(mp3wavcfg.BASE, *path_parts)
try:
return os.stat(real_file.encode("utf-8")), real_file
except FileNotFoundError:
......@@ -269,6 +347,12 @@ def handle_directory(environ, start_response, ph, real_dir):
res.write(" \"http://www.w3.org/TR/html4/strict.dtd\">\n")
res.write("<html>")
res.write("<head>")
res.write("<script src='")
res.write(environ["SCRIPT_NAME"])
res.write("/mp3wavweb/js/jquery-2.1.0.min.js'></script>")
res.write("<script src='")
res.write(environ["SCRIPT_NAME"])
res.write("/mp3wavweb/js/mp3wavweb.js'></script>")
res.write("</head>")
res.write("<body>")
parts = ph.path_parts()
......@@ -325,7 +409,8 @@ def handle_directory(environ, start_response, ph, real_dir):
def handle_file(environ, start_response, ph, real_file):
content_type = {".wav": "audio/vnd.wave",
".mp3": "audio/mpeg"}.get(ph.extension())
".mp3": "audio/mpeg",
".js": "application/javascript"}.get(ph.extension())
if content_type is None:
start_response("403 Forbidden", [])
return []
......
$( document ).ready(function() {
$(".convert").click(function() {
var original_url = $(this).attr("href");
var get_progress = function(url, context) {
$.ajax({
type: "POST",
url: url,
dataType: "json",
context: context,
success: progress,
});
}
var progress = function(json_data) {
if (json_data.status === "progress" || json_data.status === "enqueued")
{
$(this).attr("value", json_data.progress);
$(this).attr("max", json_data.workload);
get_progress(original_url + "_cont/" + json_data.progress + "/" + json_data.workload, this);
}
else if (json_data.status === "done")
{
var a = "<a href=\"" + json_data.link + "\">.mp3</a>";
var sz = " (" + json_data.size + ")";
$(a).replaceAll(this)
.css("background-color", "yellow")
.after(sz);
}
}
var progressbar = "<progress value='0' max='100'></progress>";
get_progress(original_url + "_start",
$(progressbar).replaceAll(this))
return false;
})
});
......@@ -13,13 +13,96 @@ import mp3wavcfg
def listener_fileno(self):
return self._listener._socket.fileno()
queue = []
progress = 0.0
class Entry:
def __init__(self, fn, workload):
self.__fn = fn
self.__recipients = {}
self.__workload = workload
def subscribe(self, client):
self.__recipients[client] = True
def send(self, verb, *args):
to_remove = []
for client, v in self.__recipients.items():
try:
client.send((verb, self.__fn) + tuple(args))
except BrokenPipeError:
to_remove.append(client)
for client in to_remove:
del self.__recipients[client]
def fn(self):
return self.__fn
def progress_to_mb(self, progress):
return (progress * self.__workload) // 100
def handle_progress(self, progress, before_me):
including_me = before_me + self.__workload
self.send("progress", progress, including_me)
return including_me
def workload(self):
return self.__workload
class Queue:
def __init__(self):
self.__queue = []
self.__last_progres = 0
def add(self, fn, workload):
total_workload_before = -self.__last_progres
for entry in self.__queue:
total_workload_before += entry.workload()
if entry.fn() == fn:
return total_workload_before
e = Entry(fn, workload)
self.__queue.append(e)
return total_workload_before + workload
def subscribe(self, fn, client):
workload = 0
for entry in self.__queue:
workload += entry.workload()
if fn == entry.fn():
entry.subscribe(client)
return self.__last_progres, workload
def send(self, verb, *args):
self.__queue[0].send(verb, *args)
def pop(self):
self.__queue = self.__queue[1:]
self.__last_progres = 0
def empty(self):
return len(self.__queue) == 0
def files(self):
return [e.fn() for e in self.__queue]
def peek(self):
if len(self.__queue) == 0:
return None
return self.__queue[0]
def handle_progress(self, fn, progress):
first = queue.peek()
if first.fn() != fn:
return
self.__last_progres = first.progress_to_mb(progress)
acc = 0
for entry in self.__queue:
acc = entry.handle_progress(self.__last_progres, acc)
queue = Queue()
proc = None
def work():
global proc
global queue
setattr(multiprocessing.connection.Listener, "fileno", listener_fileno)
......@@ -39,11 +122,13 @@ def work():
elif proc is not None and sock is proc.sentinel:
proc.join()
if proc.exitcode == 0:
queue[0][1].send(("done", queue[0][0]))
queue.send("done")
else:
queue[0][1].send(("fail", queue[0][0]))
queue.send("fail")
proc = None
queue = queue[1:]
queue.pop()
if queue.peek() is not None:
queue.handle_progress(0, queue.peek().fn())
start_encoding()
else:
try:
......@@ -53,47 +138,68 @@ def work():
else:
handle_message(sock, msg)
def exists(fn):
return os.path.isfile(os.path.join(mp3wavcfg.BASE, fn).encode("utf-8"))
def real_filename(fn, cache):
if cache:
return os.path.join(mp3wavcfg.BASE, "mp3-cache", fn)
else:
return os.path.join(mp3wavcfg.BASE, fn)
def size_mb(fn, cache):
try:
status = os.stat(real_filename(fn, cache).encode("utf-8"))
except FileNotFoundError:
return None
return status.st_size // (1024 * 1024)
def mp3_exists(fn):
for cache in [True, False]:
if size_mb(fn + ".mp3", cache) is not None:
return True
return False
def handle_message(client, msg):
global queue
global progress
if msg[0] == "ping":
client.send(("pong", ))
elif msg[0] == "peek":
client.send(("status", [q[0] for q in queue], progress))
elif msg[0] == "encode":
if msg[0] == "encode":
fn = msg[1]
if exists(fn + ".wav"):
if exists(fn + ".mp3"):
client.send(("already-there", fn))
else:
queue.append((fn, client))
start_encoding()
else:
sz = size_mb(fn + ".wav", False)
if sz is None:
client.send(("enofile", fn))
elif mp3_exists(fn):
client.send(("already-there", fn))
else:
workload = queue.add(fn, sz)
client.send(("enqueued", fn, workload))
start_encoding()
elif msg[0] == "progress":
progress = msg[1]
queue[0][1].send(("progress", progress))
# The "progress" message is sent from the encode() process.
# It has two arguments: the file name (sans prefix and suffix,
# and the progress in percent (0-100) as an integer.
queue.handle_progress(msg[1], msg[2])
elif msg[0] == "subscribe":
fn = msg[1]
res = queue.subscribe(fn, client)
if res is None:
if size_mb(fn + ".wav", False) is None:
client.send(("enofile", fn))
elif size_mb(fn + ".mp3", True) is None:
client.send(("noqueue", fn))
else:
client.send(("done", fn))
else:
client.send(("badreq", msg[0]))
def start_encoding():
global proc
global queue
if proc is not None:
return
if len(queue) == 0:
if queue.empty():
return
proc = multiprocessing.Process(target=encode, args=(queue[0][0], ))
proc = multiprocessing.Process(target=encode, args=(queue.peek().fn(), ))
proc.start()
def encode(fn):
c = multiprocessing.connection.Client(mp3wavcfg.socketpath, 'AF_UNIX')
result_file = os.path.join(mp3wavcfg.BASE, "mp3-cache", fn + ".mp3")
result_file = real_filename(fn + ".mp3", True)
result_dir = os.path.dirname(result_file)
try:
os.makedirs(result_dir.encode("utf-8"))
......@@ -102,7 +208,7 @@ def encode(fn):
raise
cmd = ["lame", "--preset", "standard", "--nohist",
os.path.join(mp3wavcfg.BASE, fn + ".wav").encode("utf-8"),
real_filename(fn + ".wav", False).encode("utf-8"),
(result_file + ".tmp").encode("utf-8")]
lame = subprocess.Popen(args=cmd, bufsize=0, stdin=subprocess.DEVNULL,
stderr=subprocess.PIPE)
......@@ -114,7 +220,7 @@ def encode(fn):
break
m = percent_match.search(str(msg))
if m is not None:
c.send(("progress", m.group(1)))
c.send(("progress", fn, int(m.group(1))))
x = lame.wait()
if x == 0:
os.rename((result_file + ".tmp").encode("utf-8"),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment