Commit f84030fe authored by Michael Murtaugh's avatar Michael Murtaugh
Browse files

scons!

parent d029d6e3
#!/usr/bin/env python
# coding: utf-8
import json, os, datetime
from urllib.parse import quote as urlquote
import subprocess, re, json
import sys
import re, math
import jinja2, html5lib, isodate, markdown
from indexalist.timecode import timecode_fromsecs
from indexalist.ffmpeginfo import get_info as ffmpeg_get_info
from indexalist.pdfinfo import get_info as pdf_get_info
from indexalist.imageinfo import get_info as image_get_info
# from etreeutils import innerHTML
from xml.etree import ElementTree as ET
from jinja2 import Markup
# from gitignore_parser import parse_gitignore
# import gitignore_parser
# def parse_gitignore_string (str, base_dir):
# """ code based on parse_gitignore, adapted for a string """
# rules = []
# counter = 0
# for line in str.splitlines():
# counter += 1
# line = line.rstrip('\n')
# rule = gitignore_parser.rule_from_pattern(line, base_path=gitignore_parser.Path(base_dir).resolve(),
# source=("__str__", counter))
# if rule:
# rules.append(rule)
# print (f"rule {rule}")
# if not any(r.negation for r in rules):
# return lambda file_path: any(r.match(file_path) for r in rules)
# else:
# # We have negation rules. We can't use a simple "any" to evaluate them.
# # Later rules override earlier rules.
# return lambda file_path: gitignore_parser.handle_negation(file_path, rules)
def innerHTML (elt):
if elt.text != None:
ret = elt.text
else:
ret = u""
return ret + u"".join([ET.tostring(x, method="html", encoding="unicode") for x in elt])
# def ls (path):
# # TODO: honor .indexignore's
# if path == "":
# path = "."
# ret = os.listdir(path)
# ret = [(name.lower(), name) for name in ret]
# ret.sort()
# ret = [x for _, x in ret if not (x.startswith(".") or x == "index.html")]
# return ret
# def lsdeps (path):
# def xf (p):
# if os.path.isdir(os.path.join(path, p)):
# return os.path.join(p, ".index", ".index.json")
# return p
# return list(map(xf, ls(path)))
def make_folder_meta (target, source, env):
"""
Main Builder Action, constructs .index/.index.json from:
* contained files themselves, e.g. [ *FILE*, ... ]
* file metadata, e.g. [ *.index/FILE/metadata.json*, ... ]
* contained folders via their [ *FOLDER/.index/.index.json*, ... ]
"""
# siteroot = env.Dictionary()['siteroot']
tpath = target[0].path
path = os.path.dirname(os.path.dirname(tpath))
d = {}
d['type'] = "directory"
d['filename'] = os.path.split(path)[1]
d['id'] = urlquote(d['filename'])+"/"
files = 0
folders = 0
total_bytes = 0
total_files = 0
total_folders = 0
total_files_by_ext = {}
last_modification = None
cc = []
contents_by_id = {}
meta_patches = []
description = None
for src in source:
if os.path.basename(src.path) == "description.json":
with open (src.path) as f:
description = json.load(f)
for meta in description:
if 'id' in meta:
if meta['id'] == '':
d['description'] = meta['description']
meta_patches.append((meta['id'], meta))
elif os.path.basename(src.path) == ".index.json":
folders += 1
cd = {}
cc.append(cd)
cd['type'] = "directory"
with open(src.path) as cdinfof:
cdinfo = json.load(cdinfof)
cd['id'] = cdinfo['id']
contents_by_id[cd['id']] = cd
cd['filename'] = cdinfo['filename']
# inherit / copy the description
# if 'description' in cdinfo:
# cd['description'] = cdinfo['description']
cd['total_bytes'] = cdinfo['total_bytes']
cd['total_files'] = cdinfo['total_files']
cd['total_folders'] = cdinfo['total_folders']
total_bytes += cdinfo['total_bytes']
total_folders += cdinfo['total_folders']
total_files += cdinfo['total_files']
for ext in cdinfo['total_files_by_ext']:
if ext not in total_files_by_ext:
total_files_by_ext[ext] = {'count': 0, 'total_bytes': 0}
total_files_by_ext[ext]['count']+=cdinfo['total_files_by_ext'][ext]['count']
total_files_by_ext[ext]['total_bytes']+=cdinfo['total_files_by_ext'][ext]['total_bytes']
cd['link'] = urlquote(os.path.relpath(src.path, path))
cd['last_modification'] = cdinfo['last_modification']
cd_mtime_dt = datetime.datetime.fromisoformat(cdinfo['last_modification'])
if last_modification is None or cd_mtime_dt > last_modification:
last_modification = cd_mtime_dt
elif (os.path.dirname(src.path) == path):
files += 1
cd = {}
cc.append(cd)
cd['type'] = "file"
cd['filename'] = os.path.split(src.path)[1]
cd['id'] = urlquote(cd['filename'])
contents_by_id[cd['id']] = cd
cd_mtime_dt = datetime.datetime.fromtimestamp(os.stat(src.path, follow_symlinks=False).st_mtime)
cd['mtime'] = cd_mtime_dt.isoformat()
cd['ext'] = os.path.splitext(src.path)[-1].lstrip(".").lower()
cd['size'] = os.path.getsize(src.path)
total_bytes += cd['size']
if cd['ext'] not in total_files_by_ext:
total_files_by_ext[cd['ext']] = {'count': 0, 'total_bytes': 0}
total_files_by_ext[cd['ext']]['count']+=1
total_files_by_ext[cd['ext']]['total_bytes']+=cd['size']
if last_modification is None or cd_mtime_dt > last_modification:
last_modification = cd_mtime_dt
elif src.path.endswith(".json"):
# Try to read / merge metadata
# .../.index/FILE/metadata.json
# TODO: code to match on pattern and check that file exists
# make extension handling a sub branch to handle json + md
mdir, meta_filename = os.path.split(src.path)
mdir, filename = os.path.split(mdir)
with open(src.path) as f:
meta = json.load(f)
if 'id' in meta:
# meta_by_id[meta['id']] = meta
meta_patches.append((meta['id'], meta))
else:
print (f"make_folder_meta: {path} unrecognized source file {src.path}")
# if meta_by_id:
# # Attempt to merge meta data nodes
# for meta_id in meta_by_id:
# if meta_id in contents_by_id:
# # print (f"Merging {meta_id}")
# contents_by_id[meta_id].update(meta_by_id[meta_id])
for meta_id, meta in meta_patches:
if meta_id in contents_by_id:
print (f"Merging {meta_id}, {meta}")
contents_by_id[meta_id].update(meta)
cc.sort(key=lambda x: x['filename'].lower())
d['contents'] = cc
d['total_bytes'] = total_bytes
d['files'] = files
d['folders'] = folders
d['total_folders'] = total_folders + folders # nb: totals includes this folder's direct contents as well
d['total_files'] = total_files + files
d['total_files_by_ext'] = total_files_by_ext
d['last_modification'] = last_modification.isoformat()
os.makedirs(os.path.split(tpath)[0], exist_ok=True)
with open(tpath, "w") as f:
json.dump(d, f, indent=2)
FolderMeta = Builder(action=make_folder_meta)
def ffmpeg_get_meta (target, source, env):
d = ffmpeg_get_info(source[0].path)
d['id'] = urlquote(os.path.basename(source[0].path))
def relurl(x):
return urlquote(os.path.relpath(str(x), os.path.dirname(os.path.dirname(os.path.dirname(target[0].path)))))
for s in source[1:]:
basename = os.path.basename(str(s))
if basename == "thumb.png":
d['thumbnail'] = relurl(s)
elif basename == "play.mp4":
d['play'] = relurl(s)
elif basename == "play.mp3":
d['play'] = relurl(s)
elif basename == "poster.png":
d['poster'] = relurl(s)
os.makedirs(os.path.dirname(target[0].path), exist_ok=True)
with open(target[0].path, "w") as f:
json.dump(d, f, indent=2)
FFMPEGMeta = Builder(action=ffmpeg_get_meta)
VideoPoster = Builder(action="""
mkdir -p `dirname $TARGET` && \
ffmpeg -i $SOURCE -ss 3 -vframes 1 $TARGET
""".strip())
VideoThumb = Builder(action="""
mkdir -p `dirname $TARGET` && \
ffmpeg -i $SOURCE -ss 3 -vframes 1 $TARGET && \
mogrify -resize 200x200 $TARGET
""".strip())
ImagePoster = Builder(action="""
mkdir -p $TARGET.dir && \
convert $SOURCE[0] -auto-orient -resize 640x640 $TARGET
""".strip())
ImageThumb = Builder(action="""
mkdir -p $TARGET.dir && \
convert $SOURCE[0] -auto-orient -resize 200x200 $TARGET
""".strip())
def make_image_meta (target, source, env):
d = image_get_info(source[0].path)
d['id'] = urlquote(os.path.basename(source[0].path))
def relurl(x):
return urlquote(os.path.relpath(str(x), os.path.dirname(os.path.dirname(os.path.dirname(target[0].path)))))
for s in source[1:]:
basename = os.path.basename(str(s))
if basename in ("thumb.jpg", "thumb.png"):
d['thumbnail'] = relurl(s)
elif basename in ("poster.jpg", "poster.png"):
d['poster'] = relurl(s)
os.makedirs(os.path.dirname(target[0].path), exist_ok=True)
with open(target[0].path, "w") as f:
json.dump(d, f, indent=2)
ImageMeta = Builder(action=make_image_meta)
def make_pdf_meta (target, source, env):
d = pdf_get_info(source[0].path)
d['id'] = urlquote(os.path.basename(source[0].path))
def relurl(x):
return urlquote(os.path.relpath(str(x), os.path.dirname(os.path.dirname(os.path.dirname(target[0].path)))))
for s in source[1:]:
basename = os.path.basename(str(s))
if basename in ("thumb.jpg", "thumb.png"):
d['thumbnail'] = relurl(s)
elif basename in ("poster.jpg", "poster.png"):
d['poster'] = relurl(s)
os.makedirs(os.path.dirname(target[0].path), exist_ok=True)
with open(target[0].path, "w") as f:
json.dump(d, f, indent=2)
PDFMeta = Builder(action=make_pdf_meta)
def add_attribute_to_links (src, attrname, attrvalue):
t = html5lib.parseFragment(src, treebuilder="etree", namespaceHTMLElements=False)
for a in t.findall(".//a"):
a.attrib[attrname]=attrvalue
return innerHTML(t)
def template_action (target, source, env):
# print (f"template_builder_action: {target[0]} ({len(source)} sources)")
tpath, tname = os.path.split(env.Dictionary().get("index_template"))
rootpath = env.Dictionary().get("rootpath")
jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(tpath))
jenv.filters['strftime'] = lambda x, format='%Y-%m-%d %H:%M:%S': datetime.datetime.fromisoformat(x).strftime(format)
jenv.filters['filename2title'] = lambda x: os.path.splitext(x)[0].replace("_", " ")
jenv.filters['add_attribute_to_links'] = add_attribute_to_links
md = markdown.Markdown(extensions=['meta'])
jenv.filters['markdown'] = lambda text: Markup(md.convert(text))
jenv.filters['wbr_'] = lambda x: x.replace("_", "_<wbr>")
jenv.filters['isotime'] = lambda x: datetime.datetime.fromtimestamp(t).isoformat()
jenv.filters['isoduration'] = lambda x: isodate.duration_isoformat(datetime.timedelta(0, x))
jenv.filters['timecode'] = timecode_fromsecs
template = jenv.get_template(tname)
# calc breadcrumbs...
# example breadcrumbs
# [
# {'relpath': '../../', 'name': '/'},
# {'relpath': '../', 'name': 'Constant_V'},
# {'relpath': '', 'name': 'videos'},
# ]
path = os.path.dirname(os.path.dirname(source[0].path))
rpath = os.path.relpath(os.path.abspath(path), rootpath)
if rpath == ".":
bc = [{'relpath': '', 'name':'/'}]
else:
rpath = rpath.split(os.sep)
rpath.insert(0, '/')
bc = [{'relpath': '../'*(len(rpath)-i-1), 'name': name} for i, name in enumerate(rpath)]
# print (f"path: {path}, breadcrumbs: {bc}")
# combine source json
with open(source[0].path) as fin:
data = json.load(fin)
# calc allkeys
allkeys = set()
for c in data['contents']:
for key in c:
allkeys.add(key)
data['allkeys'] = allkeys
data['breadcrumbs'] = bc
with open(target[0].path, "w") as fout:
print (template.render(**data), file=fout)
Template = Builder(action=template_action)
Template = Builder(action=Action(template_action, varlist=("index_template", "rootpath")))
### Perform the build
builders = {
"FolderMeta": FolderMeta,
"VideoPoster": VideoPoster,
"VideoThumb": VideoThumb,
"ImageThumb": ImageThumb,
"ImagePoster": ImagePoster,
"PDFMeta": PDFMeta,
"ImageMeta": ImageMeta,
"Template": Template,
"FFMPEGMeta": FFMPEGMeta
}
env = Environment(BUILDERS=builders)
# env.Append(BUILDERS=builders)
# def decide_if_file_not_exists (dependency, target, prev_ni, repo_node=None):
# return not os.path.exists(str(target))
# env2 = Environment(BUILDERS=builders)
# env2 = env.Clone()
# env2.Decider(decide_if_file_not_exists)
# def metadata_for_folder (p):
# if os.path.isdir(os.path.join(path, p)):
# return os.path.join(p, ".index", ".index.json")
# return p
# def ignore_p (path, ignores):
# path = os.path.abspath(path)
# for ignore in ignores:
# if ignore(path):
# print (f"IGNORING {path}", file=sys.stderr)
# return True
# return False
# DEFAULT_GITIGNORE_RULES = """
# /.*
# """.strip()
def ignore_p (path, ignores):
filename = os.path.basename(path)
if filename.startswith("."):
return True
if filename in ("index.html", "description.json", "__pycache__", "venv"):
return True
return False
def depwalk (path, base_path=None, ignores=None):
if base_path is None:
base_path = os.path.abspath(path)
# if ignores is None:
# ignores = []
# ignores.append(parse_gitignore_string(DEFAULT_GITIGNORE_RULES, base_path))
deps = []
# ii = os.path.join(path, ".indexignore")
# if os.path.exists(ii):
# ignores.append(parse_gitignore(os.path.abspath(ii)))
dp = os.path.join(path, "description.json")
if os.path.exists(dp):
deps.append(dp)
paths = []
for filename in os.listdir(path):
cpath = os.path.join(path, filename)
if not ignore_p(cpath, ignores):
if os.path.isfile(cpath):
deps.append(cpath)
else:
paths.append(cpath)
deps.append(os.path.join(cpath, ".index", ".index.json"))
yield (path, deps)
for cpath in paths:
for path, deps in depwalk(cpath, base_path=base_path, ignores=ignores):
yield path, deps
AUDIO_EXTENSIONS = ("mp3", "wav", "ogg", "oga", "m4a", "opus")
PDF_EXTENSIONS = ("pdf")
VIDEO_EXTENSIONS = ("mp4", "m4v", "ogv", "mpeg", "webm", "mkv", "avi", "mov", "flv", )
IMAGE_EXTENSIONS = ("jpg", "jpeg", "tiff", "tif", "png", "gif", "svg", "webp")
rootdir = Dir('#')
# print (f"root {rootdir.abspath}")
# Walk the directory tree (top-down) to express targets + dependencies
for folder, deps in depwalk("."):
# print (f"path {path}, deps: {deps}")
# for base, dirs, files in os.walk("."):
# dirs[:] = [d for d in dirs if not d.startswith(".")]
# if os.path.isdir(base):
# folder = base
# deps = [os.path.join(folder, x) for x in lsdeps(folder)]
folder_meta_path = os.path.join(folder, ".index", ".index.json")
print (f"directory:{folder} ({folder_meta_path})")
print (f" deps:{deps}")
# print ()
depsmeta = []
for d in deps:
ext = os.path.splitext(d)[1].lstrip(".").lower()
file_meta = os.path.join(folder, ".index", os.path.basename(d), "metadata.json")
if ext in AUDIO_EXTENSIONS:
meta_sources = [d]
play = os.path.join(folder, ".index", os.path.basename(d), "play.mp3")
if os.path.exists(play):
meta_sources.append(play)
env.FFMPEGMeta(target=file_meta, source=meta_sources)
depsmeta.append(file_meta)
elif ext in VIDEO_EXTENSIONS:
# if env.ENSURE_THUMBNAIL
meta_sources = [d]
poster = os.path.join(folder, ".index", os.path.basename(d), "poster.png")
if not os.path.exists(poster): # build iff it doesn't exist
env.VideoPoster(target=poster, source=d)
meta_sources.append(poster)
thumb = os.path.join(folder, ".index", os.path.basename(d), "thumb.png")
if not os.path.exists(thumb): # build iff it doesn't exist
env.VideoThumb(target=thumb, source=d)
meta_sources.append(thumb)
# play: if present include it
play = os.path.join(folder, ".index", os.path.basename(d), "play.mp4")
if os.path.exists(play):
meta_sources.append(play)
env.FFMPEGMeta(target=file_meta, source=meta_sources)
depsmeta.append(file_meta)
elif ext in IMAGE_EXTENSIONS:
meta_sources = [d]
poster = os.path.join(folder, ".index", os.path.basename(d), "poster.jpg")
if not os.path.exists(poster): # build iff it doesn't exist
env.ImagePoster(target=poster, source=d)
meta_sources.append(poster)
thumb = os.path.join(folder, ".index", os.path.basename(d), "thumb.jpg")
if not os.path.exists(thumb): # build iff it doesn't exist
env.ImageThumb(target=thumb, source=d)
meta_sources.append(thumb)
env.ImageMeta(target=file_meta, source=meta_sources)
depsmeta.append(file_meta)
elif ext in PDF_EXTENSIONS:
meta_sources = [d]
poster = os.path.join(folder, ".index", os.path.basename(d), "poster.jpg")
if not os.path.exists(poster): # build iff it doesn't exist
env.ImagePoster(target=poster, source=d)
meta_sources.append(poster)
thumb = os.path.join(folder, ".index", os.path.basename(d), "thumb.jpg")
if not os.path.exists(thumb): # build iff it doesn't exist
env.ImageThumb(target=thumb, source=d)
meta_sources.append(thumb)
env.PDFMeta(target=file_meta, source=meta_sources)
depsmeta.append(file_meta)
env.FolderMeta(target=folder_meta_path, source=deps+depsmeta)
env.Template(target=os.path.join(folder, "index.html"), \
source=[folder_meta_path, File("templates/index-template.html")], \
index_template="templates/index-template.html", \
rootpath=rootdir.abspath)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment