Commit 14ed8af0 authored by Michael Murtaugh's avatar Michael Murtaugh
Browse files

makeserver3

parent efa1113c
Pipeline #596 failed with stages
in 0 seconds
[submodule "makeserver/data/htdocs/editor/lib/ace-builds"]
path = makeserver/data/htdocs/editor/lib/ace-builds
url = https://github.com/ajaxorg/ace-builds.git
......@@ -214,7 +214,64 @@ What in a make file corresponds to the application of a file as filter.
Editor via plugin
====================
* Shadowable __lib__ folder... or else __server__, __app__ hierarchy... Do shadowing dynamically.
* Editors to offer for files in index.
* Editors as link modifiers? (javascript means of dynamically processing links?)
2020
----
makeserver3
Makeserver is an inside-out web server.
Makeserver is a scaffolding server that allows dynamic websites to be developed using a makefile.
Requires
* python 3.5
* aiohttp
To do
* (websockets-based) view to monitor output of make command
* add --js option to mixin custom javascript plugins (a la greasemonkey)
* add --saveable option to mixin saveable javascript
makewiki (aka cookbook)
Curated set of deployable makefiles
For instance to convert AVI to webm
2021
--------
Putting "code folding" into practice, merging many strands into the original project.
backend:
python / aiohttp server to serve files, run scons / terminal
(terminado / tornado?)
Bootstrap
Minimal starting point is makeserver like functionality with:
* scons
* terminal output
* (interactive terminal?!)
* (auth)
The Long term process question
Some processes take a long time to run (like a video edit, a web crawl, etc). Desire should be to initiate these processes interactively, then "detach" (in tmux style) and later rejoin, evt debug when things go awry. Detaching processes from their interface would be useful (maybe literally making implicit/automatic use of tmux/screen?). (OR maybe it's about performing the function of tmux -- ie deciding how much to buffer / or not (overcome rather than repeat the frustration of "lost" terminal output with tmux -- in ability to scrollback)...) In general, it's about really engaging with the underlying system (processes, etc) while still managing the complexity (designing) for users focussed on aspects other than the system (not "novice";).
* Adapt ptyprocess/xterm (terminado) to aiohttp
[scons use guide on tree](https://scons.org/doc/production/HTML/scons-user.html#idp140637540709104)
scons -Q -n --tree=status
......@@ -45,51 +45,7 @@ A: Not exactly, though it does incorporate the ace.js project to provide browser
Installation
-------------
Requirements / dependecies:
* python 2
* twisted
* jinja2
* ace.js (javascript editor)
Pull the submodules (ace.js)
git submodule init
git submodule update
Plus an environment with make (+ git).
pip install twisted jinja2
Installation in a virtual environment
apt-get install virtualenv python-dev
virtualenv venv
source venv/bin/activate
pip install twisted
python setup.py install
Examples (todo)
--------------
* From Directory listing to media player
* Video editing in the browser
* a Vandalist Photo gallery
TODO
---------
* Cleanup: figure out status of editor / player components
* Adding "editor by default" for all text formats is of course problematic.
* HTML/SVG: serve by default.
* CSS/JS: serve when REFERER (how to detect this though and not a link ?!)
* Add (back) edit links to a directory listing.
* GIT integration <!>
* Backend to monitor make progress / log (via a websocket?!)
TODO: Update for latest version.
Changelog
......
#!/usr/bin/python
# from cookbook import serve
from makeserver import make_server
# serve.main()
make_server.main()
# find all .md files in the directory
mdsrc=$(shell ls *.md)
# map *.mp => *.html for mdsrc
html_from_md=$(mdsrc:%.md=%.html)
all: $(html_from_md)
today:
touch `date +"%Y-%m-%d.md"`
fixnames:
rename "s/ /_/g" *
# Implicit rule to know how to make .html from .md
%.html: %.md
pandoc --from markdown \
--to html \
--standalone \
--css styles.css \
$< -o $@
# special rule for debugging variables
print-%:
@echo '$*=$($*)'
mdsrc=$(shell ls *.md)
# map *.mp => *.html for mdsrc
html_from_md=$(mdsrc:%.md=%.html)
html: $(html_from_md)
test:
echo hello world
now:
touch `date +"%Y-%m-%d-%H%M%S"`.md
%.html: %.md
pandoc --from markdown \
--to html \
--standalone \
$< -o $@
### INDEX FULES ###############
.SECONDEXPANSION:
# holy cow: this is working! (but indexalist is returning the wrong paths (with diversions at start)... need to fix this)
%/index_.html: $$(shell indexalist index --path % --format listing)
# echo 'prerequisites = $<'
indexalist index --path $* --group --template extended.html | \
html5tidy \
--indent \
--script /cookbook/draggable/draggable.js \
--script /cookbook/editor/relaymessages.js \
--style /cookbook/draggable/draggable.css \
--script /cookbook/toolpalette.js > $@
index_.html: $(shell indexalist index --path . --format listing)
indexalist index --path . --group --template extended.html | \
html5tidy \
--indent \
--script /cookbook/draggable/draggable.js \
--script /cookbook/editor/relaymessages.js \
--style /cookbook/draggable/draggable.css \
--script /cookbook/toolpalette.js > $@
###############################
%.html: %.md
pandoc --standalone $< -o $@
# special rule for debugging variables
print-%:
@echo '$*=$($*)'
mdsrc=$(shell ls *.md)
# map *.mp => *.html for mdsrc
html_from_md=$(mdsrc:%.md=%.html)
html: $(html_from_md)
### INDEX FULES ###############
%/index.html: %/*
indexical --path $* > $@
index.html: *
indexical --path . > $@
###############################
%.html: %.md
pandoc --standalone $< -o $@
# special rule for debugging variables
print-%:
@echo '$*=$($*)'
import os
from aiohttp import web
import sys, os, asyncio, json
from asyncio import create_subprocess_exec
from urllib.parse import urlparse, unquote as urlunquote, quote as urlquote
import argparse
from asyncio.subprocess import DEVNULL, STDOUT, PIPE
import aiohttp
import base64
DATAPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")
"""
Todo: use proper http error responses for errors, see aiohttp/http_exceptions.py
"""
def getDataPath (p=None):
if p:
return os.path.join(DATAPATH, p)
MAKEABLE_UP_TO_DATE = 0
MAKEABLE_NEEDS_REMAKING = 1
MAKEABLE_NOT_MAKEABLE = 2
async def is_makeable (path, cwd=".", makefile="makefile"):
"""
returns:
0=file is up to date,
1=file needs remaking,
2=file is not makeable
"""
rpath = os.path.relpath(path, cwd)
p = await create_subprocess_exec("make", "--question", "-f", makefile, rpath, cwd = cwd, stdout=DEVNULL, stderr=DEVNULL)
# retcode 0=file is up to date, 1=file needs remaking, 2=file is not makeable
ret = await p.wait()
print ("is_makeable {0}: {1}".format(path, ret), file=sys.stderr)
await log("is_makeable {0}: {1}".format(path, ret))
return ret
async def make (path, cwd=".", makefile="makefile", force=False):
rpath = os.path.relpath(path, cwd)
makeargs = ["make", "-f", makefile, rpath]
if force:
makeargs.append("-B")
p = await create_subprocess_exec(*makeargs, stdout = PIPE, stderr = STDOUT, cwd = cwd)
resp = ""
while True:
line = await p.stdout.readline()
if not line:
break
resp+=line.decode("utf-8")
# log(line)
# print (line)
ret = await p.wait() == 0
print ("make {0}: {1}".format(path, ret), file=sys.stderr)
await log("make {0}: {1}".format(path, ret))
return (ret, resp)
textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f})
is_binary_string = lambda bytes: bool(bytes.translate(None, textchars))
def is_binary_file (p):
""" returns none on ioerror """
try:
return not os.path.isdir(p) and is_binary_string(open(p, 'rb').read(1024))
except IOError:
return None
def editable (path):
_, ext = os.path.splitext(path)
ext = ext.lower()[1:]
# print (f"editable? ext {ext}", file=sys.stderr)
return ((ext not in ("html", "htm")) and not os.path.isdir(path) and not is_binary_file(path))
# From aiohttp/web_urldispatcher.py
# remove editor_url (no longer being used ?!)
def directory_as_html(filepath, directory, prefix, editor_url=None):
# returns directory's index as html
# sanity check
assert os.path.isdir(filepath)
# relative_path_to_dir = filepath.relative_to(self._directory)
relative_path_to_dir = os.path.relpath(filepath, directory)
index_of = "Index of /{}".format(relative_path_to_dir)
h1 = "<h1>{}</h1>".format(index_of)
index_list = []
dir_index = os.listdir(filepath)
for _file in sorted(dir_index):
# show file url as relative to static path
# rel_path = _file.relative_to(self._directory).as_posix()
# rel_path = os.path.relpath(_file, directory)
# file_url = prefix + '/' + rel_path
fp = os.path.join(filepath, _file)
file_url = '/' + urlquote(fp)
target = None
if editable(fp) and editor_url:
file_url = "{0}#{1}".format(editor_url, file_url)
target = "editor"
# if file is a directory, add '/' to the end of the name
if os.path.isdir(_file):
file_name = "{}/".format(_file)
else:
file_name = _file
if target:
link_src = '<li><a href="{url}" target="{target}">{name}</a></li>'.format(
url=file_url,
target="editor",
name=file_name)
else:
link_src = '<li><a href="{url}">{name}</a></li>'.format(
url=file_url,
name=file_name)
index_list.append(link_src)
ul = "<ul>\n{}\n</ul>".format('\n'.join(index_list))
body = "<body>\n{}\n{}\n</body>".format(h1, ul)
head_str = "<head>\n<title>{}</title>\n<link rel=\"stylesheet\" href=\"/include/index.css\"></script>\n<script src=\"/include/index.js\"></script>\n</head>".format(index_of)
html = "<html>\n{}\n{}\n</html>".format(head_str, body)
return html
# Based on aiohttp/web_urldispatcher.py
def directory_as_json(filepath, directory, prefix, editor_url=None):
# returns directory's index as html
# sanity check
assert os.path.isdir(filepath)
# relative_path_to_dir = filepath.relative_to(self._directory)
relative_path_to_dir = os.path.relpath(filepath, directory)
# index_of = "Index of /{}".format(relative_path_to_dir)
# h1 = "<h1>{}</h1>".format(index_of)
index_list = []
dir_index = os.listdir(filepath)
for _file in sorted(dir_index):
# show file url as relative to static path
# rel_path = _file.relative_to(self._directory).as_posix()
# rel_path = os.path.relpath(_file, directory)
# file_url = prefix + '/' + rel_path
if filepath == ".":
fp = _file
else:
fp = os.path.join(filepath, _file)
file_url = '/' + urlquote(fp)
target = None
if editable(fp) and editor_url:
file_url = "{0}#{1}".format(editor_url, file_url)
target = "editor"
# if file is a directory, add '/' to the end of the name
is_dir = os.path.isdir(fp)
if is_dir:
file_name = "{}/".format(_file)
else:
file_name = _file
stat = os.stat(fp)
index_list.append({
'url': file_url,
'name': file_name,
'editable': editable(fp),
'size': stat.st_size,
'mtime': stat.st_mtime,
'atime': stat.st_atime,
'dir': is_dir
})
ret = {}
ret['filepath'] = filepath
ret['rpath'] = relative_path_to_dir
ret['children'] = index_list
return json.dumps(ret)
async def route_get (request):
# return FileResponse(filepath, chunk_size=self._chunk_size)
# print ("rOUTE_get", request.rel_url, request.rel_url.query)
path = urlunquote(urlparse(request.rel_url.raw_path.lstrip("/")).path)
print ("GET", path, file=sys.stderr)
if path == '':
path = '.'
im = await is_makeable(path, makefile=request.app['makefile'])
remake = 'remake' in request.rel_url.query
if path and (im == MAKEABLE_NEEDS_REMAKING or \
(im == MAKEABLE_UP_TO_DATE and remake)):
code, resp = await make(path, makefile=request.app['makefile'], force=remake)
if not code:
resppage = """<!DOCTYPE html>
<head>
<meta charset="utf-8">
</head>
<body>
<p>An error occuring making the file {0}. Here's the output:</p>
<p><pre>{1}</pre></p>
</body>
</html>""".format(path, resp)
return web.Response(text=resppage, content_type="text/html")
if os.path.exists(path):
if os.path.isdir(path):
if 'application/json' in request.headers.get("Accept") or 'json' in request.rel_url.query:
return web.Response(text=directory_as_json(path, '', '', editor_url=request.app['editor']), content_type="application/json")
else:
return web.Response(text=directory_as_html(path, '', '', editor_url=request.app['editor']), content_type="text/html")
else:
return web.FileResponse(path, chunk_size=256*1024)
else:
return data
raise web.HTTPNotFound()
# return web.Response(text="404 on path {0}".format(path))
async def route_post (request):
""" write posted text value to file """
path = urlunquote(urlparse(request.rel_url.raw_path.lstrip("/")).path)
print ("POST", path, file=sys.stderr)
# if os.path.exists(path) and os.path.isfile(path):
# POST CAN CLOBBER ANY FILES THAT ARE WRITABLE BY THE USER !
# this is maybe a way too dangerous default ...
#
data = await request.post()
resp = {}
# SAVE
if 'text' in data:
text = data['text']
# doing file io inline here is not strictly speaking very async ;)
with open(path, "w") as f:
f.write(text)
resp['text'] = 'ok'
if 'base64' in data:
data = data['base64']
# print ("base64", data)
# strip optional data header: data:image/png;base64,FFFFFFFF
if data.startswith("data:"):
data = data.split(",", 1)[1]
with open(path, "wb") as f:
f.write(base64.b64decode(data))
# RENAME
if 'name' in data:
newpath = os.path.join(os.path.split(path)[0], data['name'])
if path != newpath:
os.rename(path, newpath)
resp['name'] = 'ok'
# DELETE
if 'delete' in data:
os.remove(path)
resp['delete'] = 'ok'
return web.Response(text=json.dumps(resp), content_type="application/json")
# else:
# raise web.HTTPMethodNotAllowed()
# return web.Response(text="post not allowed on {0}".format(path))
active_sockets = []
async def log (msg):
for ws in active_sockets:
await ws.send_str(msg)
async def websocket_handler(request):
print('Websocket connection starting')
ws = web.WebSocketResponse()
await ws.prepare(request)
print('Websocket connection ready')
active_sockets.append(ws)
async for msg in ws:
print(msg)
if msg.type == aiohttp.WSMsgType.TEXT:
print(msg.data)
if msg.data == 'close':
await ws.close()
else:
await ws.send_str(msg.data + '/answer')
active_sockets.remove(ws)
print('Websocket connection closed')
return ws
def main ():
ap = argparse.ArgumentParser("make & serve")
ap.add_argument("--makefile", "-f", default="Makefile")
ap.add_argument("--host", default="localhost")
ap.add_argument("--port", type=int, default=8000)
ap.add_argument("--editor", default=None)
ap.add_argument("--static", nargs=2, default=None, action="append")
args = ap.parse_args()
if sys.platform == 'win32':
# Folowing: https://docs.python.org/3/library/asyncio-subprocess.html
# On Windows, the default event loop is SelectorEventLoop which does not support subprocesses.
# ProactorEventLoop should be used instead.
loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(loop)
app = web.Application()
app['makefile'] = args.makefile
app['editor'] = args.editor
app.router.add_route('GET', '/ws', websocket_handler)
data = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")
htdocs = os.path.join(data, "htdocs")
ccurl = "__cc__"
print (f"Adding static route {ccurl}/ -> {htdocs}")
app.router.add_static(f"/{ccurl}", htdocs, show_index=True)
if args.static:
for name, path in args.static:
print ("Adding static route {0} -> {1}".format(name, path))
app.router.add_static(name, path)
app.add_routes([web.get('/{make:.*}', route_get)])
app.add_routes([web.post('/{make:.*}', route_post)])
port = args.port
tries = 0
max_tries = 10
# DYNAMIC PORT SELECTION
import socket, errno
while tries<max_tries:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.bind((args.host, port))
sock.close()
break
except socket.error as e:
if e.errno == errno.EADDRINUSE:
print (f"port {port} in use", file=sys.stderr)
port = port + 1
tries += 1
else:
raise(e)
print (f"listening on http://{args.host}:{port}/{ccurl}/")
web.run_app(app, host=args.host, port=port)
if __name__ == "__main__":
main()
from zope.interface import implements
from twisted.web.resource import IResource
from twisted.cred.portal import IRealm
class PublicHTMLRealm(object):
implements(IRealm)
def __init__(self, root):
self.root = root
def requestAvatar(self, avatarId, mind, *interfaces):
if IResource in interfaces:
return (IResource, self.root, lambda: None)
raise NotImplementedError()
#!/usr/bin/env python
import cgi
print "Content-type: text/html;charset=utf-8"
print
cgi.print_environ()
#!/usr/bin/env python
import hashlib, os, json
import cgi
import cgitb; cgitb.enable()
from uri_to_path import *
def githash (p):
# http://stackoverflow.com/questions/552659/assigning-git-sha1s-without-git
data = {}
# data['path'] = p
h = hashlib.sha1()
stat = os.stat(p)
h.update("blob {0}\0".format(stat.st_size).encode("utf-8"))
with open (p, "rb") as f:
while True:
fdata = f.read()
if not fdata: