Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
osp
tools.ethertoff
Commits
cb120305
Commit
cb120305
authored
May 27, 2019
by
gijs
Browse files
Furthered generation
parent
3cb7223f
Changes
30
Hide whitespace changes
Inline
Side-by-side
ethertoff/management/commands/generate.py
View file @
cb120305
...
...
@@ -9,6 +9,8 @@ import shutil
import
markdown
from
markdown.extensions.toc
import
TocExtension
from
py_etherpad
import
EtherpadLiteClient
from
.parse
import
parse_pads
from
.models
import
collectionFor
from
django.template
import
loader
from
django.template.defaultfilters
import
slugify
...
...
@@ -29,6 +31,7 @@ FIELD_TIME_FORMAT = '%H:%M'
import
datetime
# List pads
# Go through them, record information
# Feed content to templates
...
...
@@ -37,236 +40,6 @@ def output (path, template, context):
with
open
(
path
,
'w'
,
encoding
=
'utf-8'
)
as
w
:
w
.
write
(
loader
.
render_to_string
(
template
,
context
))
class
Model
(
object
):
fields
=
{}
def
__init__
(
self
,
key
,
**
kwargs
):
self
.
key
=
key
self
.
data
=
{}
self
.
_parse_data
(
kwargs
)
def
_parse_data
(
self
,
data
):
for
key
in
data
:
if
key
in
self
.
fields
:
self
.
data
[
key
]
=
self
.
fields
[
key
](
data
[
key
])
class
Collection
(
object
):
model
=
Model
def
get
(
self
,
key
,
invoke
=
True
):
if
key
in
self
.
index
:
return
self
.
index
[
key
]
elif
invoke
:
return
self
.
invoke
(
key
)
else
:
return
None
def
register
(
self
,
obj
):
self
.
data
.
append
(
obj
)
self
.
index
[
obj
.
key
]
=
obj
def
invoke
(
self
,
key
):
obj
=
self
.
model
(
key
)
self
.
register
(
obj
)
return
obj
class
Parser
(
object
):
def
__init__
(
self
):
self
.
fields
=
{
'produser'
:
{
'produser'
:
FIELD_SINGLE
,
'role'
:
FIELD_SINGLE
,
'biography'
:
FIELD_SINGLE
,
'event'
:
FIELD_ITERABLE
,
'trajectory'
:
FIELD_SINGLE
},
'event'
:
{
'event'
:
FIELD_SINGLE
}
}
self
.
contentTypes
=
[
'produser'
,
'bibliography'
,
'event'
,
'meeting'
,
'note'
,
'trajectory'
]
self
.
data
=
{
target
:
[]
for
target
in
self
.
contentTypes
}
self
.
index
=
{
target
:
{}
for
target
in
self
.
contentTypes
}
def
registerContentFragment
(
self
,
fragment
):
contentType
=
fragment
[
'__type__'
]
self
.
data
[
contentType
].
append
(
fragment
)
self
.
index
[
contentType
][
fragment
[
'key'
]]
=
fragment
def
getFragment
(
self
,
contentType
,
key
):
if
contentType
in
self
.
data
:
if
key
in
self
.
index
[
contentType
]:
return
self
.
index
[
contentType
][
key
]
else
:
target
=
self
.
makeContentFragmentSkeleton
(
contentType
,
key
)
self
.
registerContentFragment
(
target
)
return
target
def
asKey
(
self
,
value
):
return
slugify
(
value
)
def
makeKey
(
self
,
contentType
,
value
):
# Possibly a lookup, for now look for propery in meta
# with same name, else pick 'key' field
if
contentType
in
value
:
return
self
.
asKey
(
value
[
contentType
])
elif
'key'
in
value
:
return
self
.
asKey
(
value
[
'key'
])
else
:
return
value
[
'pk'
]
def
makeContentFragmentSkeleton
(
self
,
contentType
,
key
):
return
{
'__type__'
:
contentType
,
'key'
:
key
}
def
makeContentFragment
(
self
,
contentType
,
meta
,
body
):
key
=
self
.
makeKey
(
contentType
,
meta
)
fragment
=
self
.
getFragment
(
contentType
,
key
)
## Could be more intricate later on
for
key
in
meta
:
if
key
!=
'type'
and
key
!=
'key'
:
if
contentType
in
self
.
fields
:
if
key
in
self
.
fields
[
contentType
]:
if
self
.
fields
[
contentType
][
key
]
==
FIELD_SINGLE
:
fragment
[
key
]
=
meta
[
key
][
0
]
else
:
fragment
[
key
]
=
meta
[
key
]
else
:
fragment
[
key
]
=
meta
[
key
]
else
:
## Possibly a filter for the metadata here ?
fragment
[
key
]
=
meta
[
key
]
fragment
[
'body'
]
=
body
return
fragment
def
insertLink
(
self
,
fragment
,
prop
,
target
):
# If there is a description for the property
# follow the description: single or plural
# to extend: overwrite / datafilter
if
target
[
'__type__'
]
in
self
.
fields
:
desc
=
self
.
fields
[
target
[
'__type__'
]]
if
prop
in
desc
:
if
desc
[
prop
]
==
FIELD_SINGLE
:
target
[
prop
]
=
fragment
else
:
if
prop
not
in
target
:
target
[
prop
]
=
[]
target
[
prop
].
append
(
fragment
)
else
:
if
prop
not
in
target
:
target
[
prop
]
=
[]
target
[
prop
].
append
(
fragment
)
# Rename function
def
makeLinks
(
self
,
contentFragment
):
for
prop
in
contentFragment
:
if
prop
!=
contentFragment
[
'__type__'
]
and
prop
in
self
.
contentTypes
:
key
=
self
.
asKey
(
contentFragment
[
prop
])
target
=
self
.
getFragment
(
prop
,
key
)
self
.
insertLink
(
contentFragment
,
contentFragment
[
'__type__'
],
target
)
self
.
insertLink
(
target
,
prop
,
contentFragment
)
def
read
(
self
,
meta
,
body
):
if
'type'
in
meta
and
meta
[
'type'
]:
for
contentType
in
meta
[
'type'
]:
if
contentType
in
self
.
contentTypes
:
contentFragment
=
self
.
makeContentFragment
(
contentType
,
meta
,
body
)
self
.
makeLinks
(
contentFragment
)
# return obj
return
None
class
Field
(
object
):
def
__init__
(
self
,
raw
):
self
.
value
=
raw
def
__repr__
(
self
):
return
self
.
value
@
property
def
value
(
self
):
return
self
.
_value
@
value
.
setter
def
value
(
self
,
value
):
self
.
_value
=
self
.
parse
(
value
)
def
parse
(
self
,
raw
):
return
raw
class
SingleField
(
Field
):
def
__init__
(
self
,
raw
):
if
type
(
raw
)
is
list
:
self
.
value
=
raw
[
0
]
else
:
self
.
value
=
raw
class
DateField
(
SingleField
):
def
parse
(
self
,
value
):
return
datetime
.
datetime
.
strptime
(
value
,
FIELD_DATE_FORMAT
).
date
()
class
DateTimeField
(
SingleField
):
def
parse
(
self
,
value
):
return
datetime
.
datetime
.
strptime
(
value
,
FIELD_DATETIME_FORMAT
)
class
TimeField
(
SingleField
):
def
parse
(
self
,
value
):
return
datetime
.
datetime
.
strptime
(
value
,
FIELD_TIME_FORMAT
).
time
()
class
LookupField
(
SingleField
):
def
__init__
(
self
,
index
):
self
.
type
=
contentType
def
parse
(
self
,
value
):
if
value
:
return
index
(
self
.
contentType
).
get
(
v
),
value
[
0
])
else
:
return
None
class
IntField
(
SingleField
):
def
parse
(
self
,
value
):
return
int
(
value
)
class
MultiLookupField
(
Field
):
def
parse
(
self
,
value
):
if
value
and
type
(
value
)
is
list
:
return
[
index
(
self
.
contentType
).
get
(
v
)
for
v
in
value
]
else
:
return
None
def
lookupField
(
contentType
):
return
lambda
**
d
:
return
LookupField
(
contentType
,
**
d
)
class
Event
(
Model
):
self
.
fields
=
{
'date'
:
}
class
Produser
(
Model
):
self
.
fields
=
{
'role'
:
,
'trajectory'
:
lookupField
(
'trajectory'
)
}
pass
class
Command
(
BaseCommand
):
args
=
''
...
...
@@ -280,54 +53,26 @@ class Command(BaseCommand):
os
.
mkdir
(
outputdir
)
os
.
mkdir
(
os
.
path
.
join
(
outputdir
,
'produsers'
))
parser
=
Parser
()
epclient
=
None
for
pad
in
Pad
.
objects
.
all
():
if
not
epclient
:
epclient
=
EtherpadLiteClient
(
pad
.
server
.
apikey
,
pad
.
server
.
apiurl
)
name
,
extension
=
os
.
path
.
splitext
(
pad
.
display_slug
)
padID
=
pad
.
publicpadid
if
pad
.
is_public
else
pad
.
group
.
groupID
+
'$'
+
urllib
.
parse
.
quote
(
pad
.
name
.
replace
(
PAD_NAMESPACE_SEPARATOR
,
'_'
))
source
=
epclient
.
getText
(
padID
)[
'text'
]
print
(
'Copying static files'
)
if
extension
in
[
'.md'
,
'.markdown'
]:
md
=
markdown
.
Markdown
(
extensions
=
[
'extra'
,
'meta'
,
TocExtension
(
baselevel
=
2
),
'attr_list'
])
body
=
mark_safe
(
md
.
convert
(
source
))
try
:
meta
=
md
.
Meta
meta
[
'pk'
]
=
pad
.
pk
shutil
.
copytree
(
os
.
path
.
join
(
BASE_DIR
,
'ethertoff'
,
'templates'
,
'generated'
,
'static'
),
os
.
path
.
join
(
outputdir
,
'static'
))
if
meta
[
'type'
]
==
[
'biography'
]:
meta
[
'type'
]
=
[
'produser'
]
except
:
meta
=
{
'pk'
:
pad
.
pk
}
parser
.
read
(
meta
,
body
)
print
(
'Read {}'
.
format
(
pad
.
display_slug
))
parse_pads
()
print
(
'Read pads'
)
print
(
'Generating output'
)
output
(
os
.
path
.
join
(
outputdir
,
'produsers.html'
),
'generated/produsers.html'
,
{
'produsers'
:
sorted
(
parser
.
data
[
'produser'
],
key
=
lambda
r
:
str
(
r
[
'key'
]))
})
produsers
=
collectionFor
(
'produser'
)
events
=
collectionFor
(
'event'
)
for
produser
in
parser
.
data
[
'produser'
]:
output
(
os
.
path
.
join
(
outputdir
,
'produsers
'
,
'{}.html'
.
format
(
produser
[
'key'
])
),
'generated/produser.html'
,
{
'produser'
:
produser
})
# output(os.path.join(outputdir, 'produsers.html'), 'generated/produsers.html', { 'produsers': sorted(produsers.models, key=lambda r: str(r.key)) })
output
(
os
.
path
.
join
(
outputdir
,
'produsers
.layout.html'
),
'generated/produser
s.layout
.html'
,
{
'produser
s
'
:
sorted
(
produser
s
.
models
,
key
=
lambda
r
:
str
(
r
.
key
))
})
if
not
DEBUG
:
call_command
(
'collectstatic'
,
interactive
=
False
)
"""
Produser:
fields: {
'event': multiLookupField('event')
}
for
produser
in
produsers
.
models
:
output
(
os
.
path
.
join
(
outputdir
,
'produsers'
,
'{}.html'
.
format
(
produser
.
key
)),
'generated/produser.html'
,
{
'produser'
:
produser
})
output
(
os
.
path
.
join
(
outputdir
,
'index.html'
),
'generated/index.html'
,
{
'events'
:
sorted
(
events
.
models
,
key
=
lambda
r
:
str
(
r
.
date
),
reverse
=
True
)
})
class ProduserCollection(Collection):
model = Produser
"""
\ No newline at end of file
if
not
DEBUG
:
call_command
(
'collectstatic'
,
interactive
=
False
)
ethertoff/management/commands/links.py
0 → 100644
View file @
cb120305
from
.models
import
collectionFor
class
LinkExistsError
(
Exception
):
pass
# TODO: implement
# This error should be raised when an object is added
# to a reverse container with a different contentType.
# ContentTypes should be homogenous
class
LinkDifferentContentType
(
Exception
):
pass
class
Link
(
object
):
def
__init__
(
self
,
contentType
,
reverse
=
None
):
self
.
contentType
=
contentType
self
.
reverse
=
reverse
def
__call__
(
self
,
targetKey
,
source
):
target
=
collectionFor
(
self
.
contentType
).
get
(
targetKey
)
if
self
.
reverse
:
self
.
reverse
(
obj
=
target
,
target
=
source
)
return
target
class
MultiLink
(
Link
):
def
__call__
(
self
,
targetKeys
,
source
):
targets
=
[
collectionFor
(
self
.
contentType
).
get
(
targetKey
)
for
targetKey
in
targetKeys
]
if
self
.
reverse
:
for
target
in
targets
:
# Set the property
self
.
reverse
(
source
=
target
,
target
=
source
)
return
targets
# This couls as well be a partian
class
ReverseLink
(
object
):
def
__init__
(
self
,
name
):
self
.
linkName
=
name
def
__call__
(
self
,
obj
,
target
):
if
hasattr
(
obj
,
self
.
linkName
):
raise
LinkExistsError
()
setattr
(
obj
,
self
.
linkName
,
target
)
class
ReverseMultiLink
(
ReverseLink
):
def
__call__
(
self
,
obj
,
target
):
if
hasattr
(
target
,
self
.
linkName
):
links
=
getattr
(
obj
,
self
.
linkName
)
if
type
(
links
)
is
not
list
:
raise
LinkExistsError
else
:
links
=
getattr
(
obj
,
self
.
linkName
)
links
.
append
(
target
)
setattr
(
obj
,
self
.
linkName
,
links
)
def
is_link
(
obj
):
return
isinstance
(
obj
,
(
Link
,
MultiLink
,
ReverseLink
,
ReverseMultiLink
))
ethertoff/management/commands/models.py
0 → 100644
View file @
cb120305
FIELD_DATE_FORMAT
=
'%d-%m-%Y'
FIELD_DATETIME_FORMAT
=
'%d-%m-%Y %H:%M'
FIELD_TIME_FORMAT
=
'%H:%M'
import
datetime
import
re
# from .links import Link, MultiLink, ReverseLink, ReverseMultiLink, is_link
import
markdown
from
django.utils.safestring
import
mark_safe
def
keyFilter
(
value
):
if
type
(
value
)
is
list
:
return
'--'
.
join
([
keyFilter
(
v
)
for
v
in
value
])
elif
type
(
value
)
is
str
:
return
re
.
sub
(
r
'[^a-z0-9-]'
,
''
,
re
.
sub
(
r
'\s+'
,
'-'
,
value
.
lower
()))
else
:
return
value
class
UnknownContentTypeError
(
Exception
):
def
__init__
(
self
,
contentType
):
self
.
contentType
=
contentType
def
__str__
(
self
):
return
'Unknown contenttype `{}`'
.
format
(
self
.
contentType
)
class
LinkExistsError
(
Exception
):
pass
# TODO: implement
# This error should be raised when an object is added
# to a reverse container with a different contentType.
# ContentTypes should be homogenous
class
LinkDifferentContentType
(
Exception
):
pass
class
Link
(
object
):
def
__init__
(
self
,
contentType
,
reverse
=
None
):
self
.
contentType
=
contentType
self
.
reverse
=
reverse
def
__call__
(
self
,
targetKey
,
source
):
target
=
collectionFor
(
self
.
contentType
).
get
(
targetKey
)
if
self
.
reverse
:
self
.
reverse
(
obj
=
target
,
target
=
source
)
return
target
class
MultiLink
(
Link
):
def
__call__
(
self
,
targetKeys
,
source
):
targets
=
[
collectionFor
(
self
.
contentType
).
get
(
targetKey
)
for
targetKey
in
targetKeys
]
if
self
.
reverse
:
for
target
in
targets
:
# Set the property
self
.
reverse
(
source
=
target
,
target
=
source
)
return
targets
# This couls as well be a partian
class
ReverseLink
(
object
):
def
__init__
(
self
,
name
):
self
.
linkName
=
name
def
__call__
(
self
,
obj
,
target
):
if
hasattr
(
obj
,
self
.
linkName
):
raise
LinkExistsError
()
setattr
(
obj
,
self
.
linkName
,
target
)
class
ReverseMultiLink
(
ReverseLink
):
def
__call__
(
self
,
obj
,
target
):
if
hasattr
(
target
,
self
.
linkName
):
links
=
getattr
(
obj
,
self
.
linkName
)
if
type
(
links
)
is
not
list
:
# debug(self.linkName, obj.key, target.key, type(links))
raise
LinkExistsError
else
:
links
=
[]
links
.
append
(
target
)
setattr
(
obj
,
self
.
linkName
,
links
)
def
is_link
(
obj
):
return
isinstance
(
obj
,
(
Link
,
MultiLink
,
ReverseLink
,
ReverseMultiLink
))
class
Model
(
object
):
metadataFields
=
{}
content
=
None
keyField
=
'pk'
metadata
=
{}
def
__init__
(
self
,
key
=
None
,
metadata
=
None
,
content
=
None
):
print
(
'Keyfield {}'
.
format
(
self
.
keyField
))
if
key
:
self
.
key
=
key
else
:
self
.
key
=
keyFilter
(
metadata
[
self
.
keyField
])
if
self
.
keyField
in
metadata
else
keyFilter
(
metadata
[
'pk'
])
if
'pk'
in
metadata
else
None
self
.
metadata
=
{}
if
metadata
:
for
key
in
metadata
:
self
.
__setattr__
(
key
,
metadata
[
key
])
if
content
:
self
.
content
=
content
def
__setattr__
(
self
,
name
,
value
):
# This might break with the links
if
name
==
'key'
:
super
().
__setattr__
(
'key'
,
value
)
elif
name
==
'metadata'
:
super
().
__setattr__
(
'metadata'
,
value
)
elif
name
==
'content'
:
super
().
__setattr__
(
'content'
,
value
)
elif
name
in
self
.
metadataFields
:
print
(
name
)
if
is_link
(
self
.
metadataFields
[
name
]):
# If it is a link we also include, the obj
self
.
metadata
[
name
]
=
self
.
metadataFields
[
name
](
value
,
self
)
else
:
self
.
metadata
[
name
]
=
self
.
metadataFields
[
name
](
value
)
else
:
self
.
metadata
[
name
]
=
value
def
__getattr__
(
self
,
name
):
if
name
in
self
.
metadata
:
return
self
.
metadata
[
name
]
else
:
raise
AttributeError
()
# @property
# def content (self):
# return self._content
# @content.setter
# def contentSetter (self, content):
# self._content = content
def
__dir__
(
self
):
return
list
(
self
.
metadata
.
keys
())
+
[
'content'
]
class
Collection
(
object
):
def
__init__
(
self
,
model
):
self
.
model
=
model
self
.
models
=
[]
self
.
index
=
{}
self
.
iterindex
=
-
1
def
__iter__
(
self
):
return
self
def
__next__
(
self
):
self
.
iterindex
=
self
.
iterindex
+
1
if
len
(
self
.
models
)
>=
self
.
iterindex
:
raise
StopIteration
else
:
return
self
.
models
[
self
.
iterindex
]
"""
Retreive a model from the collection with the given key.
If instantiate is set to true an empty model will be created.
"""
def
get
(
self
,
key
,
instantiate
=
True
):
key
=
keyFilter
(
key
)
if
key
in
self
.
index
:
return
self
.
index
[
key
]
elif
instantiate
:
return
self
.
instantiate
(
key
)
else
:
return
None
"""
Register the given model with the collection
"""
def
register
(
self
,
obj
):
if
isinstance
(
obj
,
self
.
model
):
if
obj
.
key
not
in
self
.
index
:
self
.
models
.
append
(
obj
)
self
.
index
[
obj
.
key
]
=
obj
"""
Instantiate a model for the given key, metadata and content
and register it on the collection.
"""
def
instantiate
(
self
,
key
,
metadata
=
None
,
content
=
None
):
obj
=
self
.
model
(
key
=
key
,
metadata
=
metadata
,
content
=
content
)
self
.
register
(
obj
)
return
obj
def
dateField
(
value
):
return
datetime
.
datetime
.
strptime
(
value
,
FIELD_DATE_FORMAT
).
date
()
def
dateTimeField
(
value
):
def
parse
(
self
,
value
):
return
datetime
.
datetime
.
strptime
(
value
,
FIELD_DATETIME_FORMAT
)
def
timeField
(
value
):
return
datetime
.
datetime
.
strptime
(
value
,
FIELD_TIME_FORMAT
).
time
()
def
intField
(
value
):
return
int
(
value
)
def
floatField
(
value
):
return
float
(
value
)
def
stringField
(
value
):
return
str
(
value
)
def
many
(
parse
):
return
lambda
val
:
[
parse
(
v
)
for
v
in
val
]