Skip to content

GitLab

Commit cb120305 authored by gijs's avatar gijs
Browse files

Furthered generation

parent 3cb7223f
Hide whitespace changes
Inline Side-by-side
ethertoff/management/commands/generate.py
View file @ cb120305
......@@ -9,6 +9,8 @@ import shutil
import markdown
from markdown.extensions.toc import TocExtension
from py_etherpad import EtherpadLiteClient
from .parse import parse_pads
from .models import collectionFor
from django.template import loader
from django.template.defaultfilters import slugify
......@@ -29,6 +31,7 @@ FIELD_TIME_FORMAT = '%H:%M'
import datetime
# List pads
# Go through them, record information
# Feed content to templates
......@@ -37,236 +40,6 @@ def output (path, template, context):
with open(path, 'w', encoding='utf-8') as w:
w.write(loader.render_to_string(template, context))
class Model(object):
fields = {}
def __init__ (self, key, **kwargs):
self.key = key
self.data = {}
self._parse_data(kwargs)
def _parse_data (self, data):
for key in data:
if key in self.fields:
self.data[key] = self.fields[key](data[key])
class Collection(object):
model = Model
def get (self, key, invoke=True):
if key in self.index:
return self.index[key]
elif invoke:
return self.invoke(key)
else:
return None
def register (self, obj):
self.data.append(obj)
self.index[obj.key] = obj
def invoke (self, key):
obj = self.model(key)
self.register(obj)
return obj
class Parser(object):
def __init__ (self):
self.fields = {
'produser': {
'produser': FIELD_SINGLE,
'role': FIELD_SINGLE,
'biography': FIELD_SINGLE,
'event': FIELD_ITERABLE,
'trajectory': FIELD_SINGLE
},
'event': {
'event': FIELD_SINGLE
}
}
self.contentTypes = ['produser', 'bibliography', 'event', 'meeting', 'note', 'trajectory']
self.data = { target: [] for target in self.contentTypes }
self.index = { target: {} for target in self.contentTypes }
def registerContentFragment (self, fragment):
contentType = fragment['__type__']
self.data[contentType].append(fragment)
self.index[contentType][fragment['key']] = fragment
def getFragment (self, contentType, key):
if contentType in self.data:
if key in self.index[contentType]:
return self.index[contentType][key]
else:
target = self.makeContentFragmentSkeleton(contentType, key)
self.registerContentFragment(target)
return target
def asKey(self, value):
return slugify(value)
def makeKey(self, contentType, value):
# Possibly a lookup, for now look for propery in meta
# with same name, else pick 'key' field
if contentType in value:
return self.asKey(value[contentType])
elif 'key' in value:
return self.asKey(value['key'])
else:
return value['pk']
def makeContentFragmentSkeleton (self, contentType, key):
return {
'__type__': contentType,
'key': key
}
def makeContentFragment (self, contentType, meta, body):
key = self.makeKey(contentType, meta)
fragment = self.getFragment(contentType, key)
## Could be more intricate later on
for key in meta:
if key != 'type' and key != 'key':
if contentType in self.fields:
if key in self.fields[contentType]:
if self.fields[contentType][key] == FIELD_SINGLE:
fragment[key] = meta[key][0]
else:
fragment[key] = meta[key]
else:
fragment[key] = meta[key]
else:
## Possibly a filter for the metadata here ?
fragment[key] = meta[key]
fragment['body'] = body
return fragment
def insertLink (self, fragment, prop, target):
# If there is a description for the property
# follow the description: single or plural
# to extend: overwrite / datafilter
if target['__type__'] in self.fields:
desc = self.fields[target['__type__']]
if prop in desc:
if desc[prop] == FIELD_SINGLE:
target[prop] = fragment
else:
if prop not in target:
target[prop] = []
target[prop].append(fragment)
else:
if prop not in target:
target[prop] = []
target[prop].append(fragment)
# Rename function
def makeLinks(self, contentFragment):
for prop in contentFragment:
if prop != contentFragment['__type__'] and prop in self.contentTypes:
key = self.asKey(contentFragment[prop])
target = self.getFragment(prop, key)
self.insertLink(contentFragment, contentFragment['__type__'], target)
self.insertLink(target, prop, contentFragment)
def read (self, meta, body):
if 'type' in meta and meta['type']:
for contentType in meta['type']:
if contentType in self.contentTypes:
contentFragment = self.makeContentFragment(contentType, meta, body)
self.makeLinks(contentFragment)
# return obj
return None
class Field (object):
def __init__ (self, raw):
self.value = raw
def __repr__ (self):
return self.value
@property
def value (self):
return self._value
@value.setter
def value (self, value):
self._value = self.parse(value)
def parse (self, raw):
return raw
class SingleField (Field):
def __init__ (self, raw):
if type(raw) is list:
self.value = raw[0]
else:
self.value = raw
class DateField (SingleField):
def parse (self, value):
return datetime.datetime.strptime(value, FIELD_DATE_FORMAT).date()
class DateTimeField (SingleField):
def parse (self, value):
return datetime.datetime.strptime(value, FIELD_DATETIME_FORMAT)
class TimeField (SingleField):
def parse (self, value):
return datetime.datetime.strptime(value, FIELD_TIME_FORMAT).time()
class LookupField (SingleField):
def __init__ (self, index):
self.type = contentType
def parse (self, value):
if value:
return index(self.contentType).get(v), value[0])
else:
return None
class IntField (SingleField):
def parse (self, value):
return int(value)
class MultiLookupField (Field):
def parse (self, value):
if value and type(value) is list:
return [ index(self.contentType).get(v) for v in value ]
else:
return None
def lookupField(contentType):
return lambda **d: return LookupField(contentType, **d)
class Event (Model):
self.fields = {
'date':
}
class Produser (Model):
self.fields = {
'role': ,
'trajectory': lookupField('trajectory')
}
pass
class Command(BaseCommand):
args = ''
......@@ -280,54 +53,26 @@ class Command(BaseCommand):
os.mkdir(outputdir)
os.mkdir(os.path.join(outputdir, 'produsers'))
parser = Parser()
epclient = None
for pad in Pad.objects.all():
if not epclient:
epclient = EtherpadLiteClient(pad.server.apikey, pad.server.apiurl)
name, extension = os.path.splitext(pad.display_slug)
padID = pad.publicpadid if pad.is_public else pad.group.groupID + '$' + urllib.parse.quote(pad.name.replace(PAD_NAMESPACE_SEPARATOR, '_'))
source = epclient.getText(padID)['text']
print('Copying static files')
if extension in ['.md', '.markdown']:
md = markdown.Markdown(extensions=['extra', 'meta', TocExtension(baselevel=2), 'attr_list'])
body = mark_safe(md.convert(source))
try:
meta = md.Meta
meta['pk'] = pad.pk
shutil.copytree(os.path.join(BASE_DIR, 'ethertoff', 'templates', 'generated', 'static'), os.path.join(outputdir, 'static'))
if meta['type'] == ['biography']:
meta['type'] = ['produser']
except:
meta = { 'pk': pad.pk }
parser.read(meta, body)
print('Read {}'.format(pad.display_slug))
parse_pads()
print('Read pads')
print('Generating output')
output(os.path.join(outputdir, 'produsers.html'), 'generated/produsers.html', { 'produsers': sorted(parser.data['produser'], key=lambda r: str(r['key'])) })
produsers = collectionFor('produser')
events = collectionFor('event')
for produser in parser.data['produser']:
output(os.path.join(outputdir, 'produsers', '{}.html'.format(produser['key'])), 'generated/produser.html', { 'produser': produser })
# output(os.path.join(outputdir, 'produsers.html'), 'generated/produsers.html', { 'produsers': sorted(produsers.models, key=lambda r: str(r.key)) })
output(os.path.join(outputdir, 'produsers.layout.html'), 'generated/produsers.layout.html', { 'produsers': sorted(produsers.models, key=lambda r: str(r.key)) })
if not DEBUG:
call_command('collectstatic', interactive=False)
"""
Produser:
fields: {
'event': multiLookupField('event')
}
for produser in produsers.models:
output(os.path.join(outputdir, 'produsers', '{}.html'.format(produser.key)), 'generated/produser.html', { 'produser': produser })
output(os.path.join(outputdir, 'index.html'), 'generated/index.html', { 'events': sorted(events.models, key=lambda r: str(r.date), reverse=True) })
class ProduserCollection(Collection):
model = Produser
"""
\ No newline at end of file
if not DEBUG:
call_command('collectstatic', interactive=False)
ethertoff/management/commands/links.py 0 → 100644
View file @ cb120305
from .models import collectionFor
class LinkExistsError(Exception):
pass # TODO: implement
# This error should be raised when an object is added
# to a reverse container with a different contentType.
# ContentTypes should be homogenous
class LinkDifferentContentType(Exception):
pass
class Link(object):
def __init__ (self, contentType, reverse=None):
self.contentType = contentType
self.reverse = reverse
def __call__ (self, targetKey, source):
target = collectionFor(self.contentType).get(targetKey)
if self.reverse:
self.reverse(obj=target, target=source)
return target
class MultiLink(Link):
def __call__ (self, targetKeys, source):
targets = [ collectionFor(self.contentType).get(targetKey) for targetKey in targetKeys ]
if self.reverse:
for target in targets:
# Set the property
self.reverse(source=target, target=source)
return targets
# This couls as well be a partian
class ReverseLink(object):
def __init__ (self, name):
self.linkName = name
def __call__ (self, obj, target):
if hasattr(obj, self.linkName):
raise LinkExistsError()
setattr(obj, self.linkName, target)
class ReverseMultiLink(ReverseLink):
def __call__ (self, obj, target):
if hasattr(target, self.linkName):
links = getattr(obj, self.linkName)
if type(links) is not list:
raise LinkExistsError
else:
links = getattr(obj, self.linkName)
links.append(target)
setattr(obj, self.linkName, links)
def is_link (obj):
return isinstance(obj, (Link, MultiLink, ReverseLink, ReverseMultiLink))
ethertoff/management/commands/models.py 0 → 100644
View file @ cb120305
FIELD_DATE_FORMAT = '%d-%m-%Y'
FIELD_DATETIME_FORMAT = '%d-%m-%Y %H:%M'
FIELD_TIME_FORMAT = '%H:%M'
import datetime
import re
# from .links import Link, MultiLink, ReverseLink, ReverseMultiLink, is_link
import markdown
from django.utils.safestring import mark_safe
def keyFilter (value):
if type(value) is list:
return '--'.join([keyFilter(v) for v in value])
elif type(value) is str:
return re.sub(r'[^a-z0-9-]', '', re.sub(r'\s+', '-', value.lower()))
else:
return value
class UnknownContentTypeError(Exception):
def __init__(self, contentType):
self.contentType = contentType
def __str__(self):
return 'Unknown contenttype `{}`'.format(self.contentType)
class LinkExistsError(Exception):
pass # TODO: implement
# This error should be raised when an object is added
# to a reverse container with a different contentType.
# ContentTypes should be homogenous
class LinkDifferentContentType(Exception):
pass
class Link(object):
def __init__ (self, contentType, reverse=None):
self.contentType = contentType
self.reverse = reverse
def __call__ (self, targetKey, source):
target = collectionFor(self.contentType).get(targetKey)
if self.reverse:
self.reverse(obj=target, target=source)
return target
class MultiLink(Link):
def __call__ (self, targetKeys, source):
targets = [ collectionFor(self.contentType).get(targetKey) for targetKey in targetKeys ]
if self.reverse:
for target in targets:
# Set the property
self.reverse(source=target, target=source)
return targets
# This couls as well be a partian
class ReverseLink(object):
def __init__ (self, name):
self.linkName = name
def __call__ (self, obj, target):
if hasattr(obj, self.linkName):
raise LinkExistsError()
setattr(obj, self.linkName, target)
class ReverseMultiLink(ReverseLink):
def __call__ (self, obj, target):
if hasattr(target, self.linkName):
links = getattr(obj, self.linkName)
if type(links) is not list:
# debug(self.linkName, obj.key, target.key, type(links))
raise LinkExistsError
else:
links = []
links.append(target)
setattr(obj, self.linkName, links)
def is_link (obj):
return isinstance(obj, (Link, MultiLink, ReverseLink, ReverseMultiLink))
class Model(object):
metadataFields = {}
content = None
keyField = 'pk'
metadata = {}
def __init__ (self, key=None, metadata=None, content=None):
print('Keyfield {}'.format(self.keyField))
if key:
self.key = key
else:
self.key = keyFilter(metadata[self.keyField]) if self.keyField in metadata else keyFilter(metadata['pk']) if 'pk' in metadata else None
self.metadata = {}
if metadata:
for key in metadata:
self.__setattr__(key, metadata[key])
if content:
self.content = content
def __setattr__ (self, name, value):
# This might break with the links
if name == 'key':
super().__setattr__('key', value)
elif name == 'metadata':
super().__setattr__('metadata', value)
elif name == 'content':
super().__setattr__('content', value)
elif name in self.metadataFields:
print(name)
if is_link(self.metadataFields[name]):
# If it is a link we also include, the obj
self.metadata[name] = self.metadataFields[name](value, self)
else:
self.metadata[name] = self.metadataFields[name](value)
else:
self.metadata[name] = value
def __getattr__ (self, name):
if name in self.metadata:
return self.metadata[name]
else:
raise AttributeError()
# @property
# def content (self):
# return self._content
# @content.setter
# def contentSetter (self, content):
# self._content = content
def __dir__ (self):
return list(self.metadata.keys()) + ['content']
class Collection(object):
def __init__ (self, model):
self.model = model
self.models = []
self.index = {}
self.iterindex = -1
def __iter__ (self):
return self
def __next__ (self):
self.iterindex = self.iterindex + 1
if len(self.models) >= self.iterindex:
raise StopIteration
else:
return self.models[self.iterindex]
"""
Retreive a model from the collection with the given key.
If instantiate is set to true an empty model will be created.
"""
def get (self, key, instantiate=True):
key = keyFilter(key)
if key in self.index:
return self.index[key]
elif instantiate:
return self.instantiate(key)
else:
return None
"""
Register the given model with the collection
"""
def register (self, obj):
if isinstance(obj, self.model):
if obj.key not in self.index:
self.models.append(obj)
self.index[obj.key] = obj
"""
Instantiate a model for the given key, metadata and content
and register it on the collection.
"""
def instantiate (self, key, metadata=None, content=None):
obj = self.model(key=key, metadata=metadata, content=content)
self.register(obj)
return obj
def dateField (value):
return datetime.datetime.strptime(value, FIELD_DATE_FORMAT).date()
def dateTimeField (value):
def parse (self, value):
return datetime.datetime.strptime(value, FIELD_DATETIME_FORMAT)
def timeField (value):
return datetime.datetime.strptime(value, FIELD_TIME_FORMAT).time()
def intField (value):
return int(value)
def floatField(value):
return float(value)
def stringField(value):
return str(value)
def many(parse):
return lambda val: [parse(v) for v in val]