Commit cb0a1198 authored by gijs's avatar gijs
Browse files

Start of the generator.

parent 151869c3
from urllib.request import URLError, urlopen
from urllib.parse import urlencode, urljoin
import os.path
import os
import json
import datetime
CACHE_DIR = os.path.join(os.path.dirname(__file__), 'cache')
API_URL = ""
DEBUG = True
PER_PAGE = 100
def debug (msg):
......@@ -25,14 +20,16 @@ class ApiError(Exception):
return '(%s) => %s'%(self.url,self.what)
class ApiCall (object):
def __init__ (self, api_path, api_url = API_URL, cache_dir = CACHE_DIR):
def __init__ (self, api_path, query = None, api_url = API_URL, cache_dir = CACHE_DIR, paged = True):
self.paged = paged
self._cache = None
self.api_path = list(map(str, api_path))
self.api_url = api_url
self.cache_dir = cache_dir
self.query = query
def cache_file(self):
def cache_location(self):
return os.path.join(self.cache_dir, '{}.json'.format('.'.join(self.api_path)))
......@@ -41,83 +38,132 @@ class ApiCall (object):
def has_cache(self):
return os.path.exists(self.cache_file)
return os.path.exists(self.cache_location)
def invalidate_cache(self):
if self.has_cache:
def make_cache(self):
def write_cache(self, data):
obj = self.get_api()
json.dump(obj, open(self.cache_file, 'w'))
with open(self.cache_location, 'w') as h:
except ApiError as e:
'reason': e.what,
}, open(self.cache_file, 'w'))
obj = []
}, open(self.cache_location, 'w'))
data = []
self._cache = obj
self._cache = data
def load_cache(self):
def read_cache(self):
debug('Hit cache {}'.format(self.url))
if not self._cache:
obj = json.load(open(self.cache_file, 'r'))
if 'reason' in obj:
self._cache = []
self._cache = obj
with open(self.cache_location, 'r') as h:
self._cache = self.parse_cache(
return self._cache
Returns values for the call. If the request is paginated go through
all pages
def get_api(self):
page = 1
items = []
while page < MAX_PAGES:
headers, pageitems = self.get_api_page(page)
if page >= int(headers['X-Total-Pages']):
return items
page += 1
def get_api_page(self, page=0):
debug('{}, page {}'.format(self.url, page))
def get_api (self):
_, data = self.get_api_url(self.url, self.query)
return data
except ApiError:
return None
def get_api_url(self, url, query = None):
if query:
url = '{}?{}'.format(url, urlencode(query))
q = urlencode({'page': page, 'per_page': PER_PAGE})
url = '{}?{}'.format(self.url, q)
res = urlopen(url)
return ({ k: v for (k,v) in res.getheaders() }, json.loads(
return ({ k: v for (k,v) in res.getheaders() }, self.parse_api_result(
except URLError as e:
if hasattr(e, 'reason'):
raise ApiError(url, e.reason)
elif hasattr(e, 'code'):
raise ApiError(url, e.code)
# elif hasattr(e, 'code'):
# raise ApiError(url, e.code)
return None
def get (self):
if self.has_cache:
return self.load_cache()
if not self.has_cache:
data = self.get_api()
return self.read_cache()
def parse_api_result (self, raw):
return raw.decode()
def prepare_cache (self, raw):
return raw
def parse_cache (self, raw):
return raw
class ApiCallJson(ApiCall):
def parse_api_result(self, raw):
return json.loads(raw)
def prepare_cache(self, raw):
return json.dumps(raw)
def parse_cache(self, raw):
return json.loads(raw)
Returns values for the call. If the request is paginated go through
all pages
def get_api(self):
query = self.query.copy() if self.query else {}
if self.paged:
page = 1
data = []
query['per_page'] = PER_PAGE
max_pages = MAX_PAGES
while page and page < max_pages:
query['page'] = page
headers, page_data = self.get_api_url(self.url, query)
page = int(headers['X-Next-Page']) if headers['X-Next-Page'] else None
except ApiError:
return data
return self.load_cache()
query['per_page'] = 1
_, data = self.get_api_url(self.url, query)
return data
class ApiCallRaw (ApiCall):
multi_page = False
def cache_location(self):
return os.path.join(self.cache_dir, '{}.data'.format('.'.join(self.api_path)))
def get_group (group_id):
return ApiCallJson(['groups', group_id], paged=False)
def get_project (project_id):
return ApiCallJson(['projects', project_id], paged=False)
# A way to make a call, cache and be able te remove the cache when needed
# prepare a call, set of url and local cache file
def group_projects (group = GROUP_ID):
return ApiCall(['groups', group, 'projects'])
def get_projects (group_id):
return ApiCallJson(['groups', group_id, 'projects'])
def get_commits (project_id):
return ApiCallJson(['projects', project_id, 'repository', 'commits'])
def commits (project_id = None):
if project_id is not None:
return ApiCall(['projects', project_id, 'repository', 'commits'])
def get_tree (project_id, path=None):
query = { 'path': path } if path else None
return ApiCallJson(['projects', project_id, 'repository', 'tree'], query=query)
def tree (project_id = None):
if project_id is not None:
return ApiCall(['projects', project_id, 'repository', 'tree'])
def get_raw (project_id, file_id):
return ApiCallRaw(['projects', project_id, 'repository', 'blobs', file_id, 'raw'])
\ No newline at end of file
import api
from jinja2 import Environment, FileSystemLoader, select_autoescape
import time
from models import Project
from models import Group
from settings import GROUP_ID, TEMPLATE_DIR, OUTPUT_DIR
import os.path
from os import makedirs
env = Environment(
autoescape=select_autoescape(['html', 'xml'])
group = Group(GROUP_ID)
# print(group)
# for project in group.projects:
# print(hasattr(project, 'tree'))
# print(project.links)
# print(
# print(project.tree)
# print(group.projects.models)
def get_projects ():
call = api.group_projects()
return call.get()
# def get_projects ():
# call = api.group_projects(group_id = GROUP_ID)
# return call.get()
def get_tree (project_id):
call = api.tree(project_id)
return call.get()
# def get_tree (project_id):
# call = api.tree(project_id)
# return call.get()
def get_commits (project_id):
call = api.commits(project_id)
return call.get()
# def get_commits (project_id):
# call = api.commits(project_id)
# return call.get()
def make_local_url (url):
return url
# def make_local_url (url):
# return url
def parse_project (project):
return Project(id=project['id'], tree = get_tree(project['id']), commits = get_commits(project['id']))
# def parse_project (project):
# return Project(id=project['id'], name=project['name'], tree = get_tree(project['id']), commits = get_commits(project['id']))
print('Loading templates')
projects = [parse_project(project) for project in get_projects()]
# print('Loading templates')
# projects = [parse_project(project) for project in get_projects()]
print('Starting generation')
template = env.get_template('projects.html')'output/index.html'), 'index.html'))
print('Generating individual project pages')
project_template = env.get_template('project.html')
for project in group.projects:
project_url = project.links['self']
project_name = os.path.basename(project_url)
project_path = os.path.dirname(project_url)
output_path = os.path.join(OUTPUT_DIR, project_path)
for project in projects:'output/{}.html'.format(
if not os.path.exists(output_path):
makedirs(output_path), project_name))
print('Generation finished')
def update_project(project_id):
call = get_commits(project_id)
\ No newline at end of file
# def update_project(project_id):
# call = get_commits(project_id)
# call.invalidate_cache()
# call.get()
\ No newline at end of file
import utils
from utils import make_local_url
from api import ApiCallJson, ApiCallJson, ApiCall, get_commits, get_group, get_projects, get_project, get_tree
import os.path
import markdown
class Project (object):
def __init__ (self, id=None, name=None, tree=[], readme=None, commits = []): = id, = name,
self.tree = tree,
self.readme = readme,
self.commits = commits
- Api Call encoded in the 'model'
- Possibility to avoid cache
- constructor inserts data from the API
- way to retreive attributes: data, __getter__ () => key in data return
- 'get', get on the api Call
class Model (object):
def __init__ (self, id, parent_id=None, data={}):
if id: = id
raise ValueError("Need an id.") = data
if parent_id:
self.parent_id = parent_id = data
def api_call (self):
raise NotImplementedError()
def get (self):
data = self.api_call.get()
if data:
for k, v in data.items():
self.__setattr__(k, v)
def __setattr__ (self, name, value):
if name == 'data' or name == 'id':
super().__setattr__(name, value)
else:[name] = value
def __getattr__ (self, name):
if name in
return super().__getattr__(name)
class Collection (object):
model = Model
def __init__ (self, models = []):
self.models = []
for model in models:
def insert (self, model):
if not isinstance(model, self.model):
model = self.model(model['id'], data=model)
def __iter__ (self):
return iter(self.models)
def __len__ (self):
return len(self.models)
class Commit (Model):
# @property
# def api_call (self):
# return ApiCallJson(['project', self.project_id, 'commits',])
class Project (Model):
def links (self):
# Derive the URL for the project from the name.
# Split on '.', remove last (filename)
# Remove first if it is OSP (old naming scheme)
parts ='.')
path = parts[:-1]
name = '{}.html'.format(parts[-1])
if not path:
# Assume work if no prefix was found
path = ['work']
elif path[0] == 'osp':
return {
'self': utils.make_local_url('{}.html'.format(
\ No newline at end of file
'self': make_local_url(os.path.join(*path, name))
def commits (self):
if not hasattr(self, '_commits'):
self._commits = Commits(get_commits(
return self._commits
def tree (self):
if not hasattr(self, '_tree'):
self._tree = Tree(get_tree(
return self._tree
def readme (self):
# Return a readme
def api_url (self):
return get_project(
class Projects (Collection):
model = Project
class Group (Model):
def api_call (self):
return get_group(
def projects (self):
if not hasattr(self, '_projects'):
self._projects = Projects(get_projects(
return self._projects
class Commits (Collection):
model = Commit
Wrapper around a folder from the api.
- can display the foldername
- an iterator through the contents
# class TreeFolder(object):
# pass
# class TreeFile():
# def __init__ (self, name, data):
# = name
# = data
# def __str__ (self):
# return
# class TreeFileMarkdown(TreeFile):
# def __str__ (self):
# return markdown.markdown(
# class TreeFileImage(TreeFile):
# def __str__ (self):
# return self.path
# def thumbnail(width):
# pass
class TreeEntry (Model):
class Tree (Collection):
model = TreeEntry
# class TreeEntry (object):
# props = ['id', 'name', 'type', 'path', 'mode', 'project']
# def __init__ (self, **kwargs):
# for k in kwargs:
# if k in self.props:
# self.__setattr__(k, kwargs[k])
# def __str__ (self):
# return self.get()
# def get (self):
# print('Trying to load',
# return raw(,
\ No newline at end of file
import os.path
BASE_DIR = os.path.dirname(__file__)
CACHE_DIR = os.path.join(BASE_DIR, 'cache')
TEMPLATE_DIR = os.path.join(BASE_DIR, 'templates')
OUTPUT_DIR = os.path.join(BASE_DIR, 'output')
API_URL = ""
DEBUG = False
PER_PAGE = 100
\ No newline at end of file
{{ }}
{% if project.readme %}
{{ project.readme }}
{% endif %}
{% for file in project.tree %}
<li>{{ }}</li>{% endfor %}
......@@ -6,5 +9,6 @@
{% for commit in project.commits %}
<li>{{ commit.message }}</li>{% endfor %}
<li>{{ commit.author_name }}: {{ commit.message }}</li>
{% endfor %}
\ No newline at end of file
{% for project in projects %}
<li><a href="{{ project.links.self }}">{{ }}</a></li>{% endfor %}
<li><a href="{{ project.links.self }}">{{ }} ({{ project.commits|length }} commits)</a></li>{% endfor %}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment