Commit f96b035a authored by gijs's avatar gijs
Browse files

First working version of the generator

parent b5471044
*.pyc
__pycache__
output/*
cache/*
\ No newline at end of file
from urllib.request import URLError, urlopen
from urllib.parse import urlencode, urljoin
import os.path
import os
import json
import datetime
CACHE_DIR = os.path.join(os.path.dirname(__file__), 'cache')
API_URL = "https://gitlab.constantvzw.org/api/v4/"
GROUP_ID = 8
DEBUG = True
MAX_PAGES = 100
PER_PAGE = 100
def debug (msg):
if DEBUG:
print(msg)
class ApiError(Exception):
def __init__(self, url, what):
self.url = url
self.what = what
def __str__(self):
return '(%s) => %s'%(self.url,self.what)
class ApiCall (object):
def __init__ (self, api_path, api_url = API_URL, cache_dir = CACHE_DIR):
self._cache = None
self.api_path = list(map(str, api_path))
self.api_url = api_url
self.cache_dir = cache_dir
@property
def cache_file(self):
return os.path.join(self.cache_dir, '{}.json'.format('.'.join(self.api_path)))
@property
def url(self):
return urljoin(self.api_url, '/'.join(self.api_path))
@property
def has_cache(self):
return os.path.exists(self.cache_file)
def invalidate_cache(self):
if self.has_cache:
os.unlink(self.cache_file)
def make_cache(self):
try:
obj = self.get_api()
json.dump(obj, open(self.cache_file, 'w'))
except ApiError as e:
json.dump({
'reason': e.what,
'timestamp': datetime.datetime.now().timestamp()
}, open(self.cache_file, 'w'))
obj = []
self._cache = obj
def load_cache(self):
debug('Hit cache {}'.format(self.url))
if not self._cache:
obj = json.load(open(self.cache_file, 'r'))
if 'reason' in obj:
self._cache = []
else:
self._cache = obj
return self._cache
"""
Returns values for the call. If the request is paginated go through
all pages
"""
def get_api(self):
page = 1
items = []
while page < MAX_PAGES:
headers, pageitems = self.get_api_page(page)
items.extend(pageitems)
if page >= int(headers['X-Total-Pages']):
return items
else:
page += 1
def get_api_page(self, page=0):
debug('{}, page {}'.format(self.url, page))
try:
q = urlencode({'page': page, 'per_page': PER_PAGE})
url = '{}?{}'.format(self.url, q)
res = urlopen(url)
return ({ k: v for (k,v) in res.getheaders() }, json.loads(res.read()))
except URLError as e:
if hasattr(e, 'reason'):
raise ApiError(url, e.reason)
elif hasattr(e, 'code'):
raise ApiError(url, e.code)
return None
def get (self):
if self.has_cache:
return self.load_cache()
else:
self.make_cache()
return self.load_cache()
# A way to make a call, cache and be able te remove the cache when needed
# prepare a call, set of url and local cache file
def group_projects (group = GROUP_ID):
return ApiCall(['groups', group, 'projects'])
def commits (project_id = None):
if project_id is not None:
return ApiCall(['projects', project_id, 'repository', 'commits'])
def tree (project_id = None):
if project_id is not None:
return ApiCall(['projects', project_id, 'repository', 'tree'])
import api
from jinja2 import Environment, FileSystemLoader, select_autoescape
import time
from models import Project
env = Environment(
loader=FileSystemLoader('templates'),
autoescape=select_autoescape(['html', 'xml'])
)
def get_projects ():
call = api.group_projects()
return call.get()
def get_tree (project_id):
call = api.tree(project_id)
return call.get()
def get_commits (project_id):
call = api.commits(project_id)
return call.get()
def make_local_url (url):
return url
def parse_project (project):
return Project(id=project['id'], tree = get_tree(project['id']), commits = get_commits(project['id']))
print('Loading templates')
projects = [parse_project(project) for project in get_projects()]
print('Starting generation')
template = env.get_template('projects.html')
template.stream(projects=projects).dump('output/index.html')
project_template = env.get_template('project.html')
for project in projects:
project_template.stream(project=project).dump('output/{}.html'.format(project.name))
print('Generation finished')
def update_project(project_id):
call = get_commits(project_id)
call.invalidate_cache()
call.get()
\ No newline at end of file
import utils
class Project (object):
def __init__ (self, id=None, name=None, tree=[], readme=None, commits = []):
self.id = id,
self.name = name,
self.tree = tree,
self.readme = readme,
self.commits = commits
@property
def links (self):
return {
'self': utils.make_local_url('{}.html'.format(self.name))
}
\ No newline at end of file
// deal with api error
// count attempts ? perhaps record timestamp
// 404 → cache
{
'error': 404,
'timestamp': x
}
tree: [ {
"id": "591330eb757b8f3da46258f89b530bdac7c92e17",
"name": "contour.html",
"type": "blob",
"path": "contour.html",
"mode": "100644"
}
]
commits: [ {
"id": "e14c0a584be4dc086fc48938b848a16003cd22ea",
"short_id": "e14c0a58",
"created_at": "2019-05-03T12:48:33.000Z",
"parent_ids": [],
"title": "A new project: the remake of OSP website!",
"message": "A new project: the remake of OSP website!\n",
"author_name": "Stephanie Vilayphiou",
"author_email": "stephanie@stdin.fr",
"authored_date": "2019-05-03T12:48:33.000Z",
"committer_name": "Stephanie Vilayphiou",
"committer_email": "stephanie@stdin.fr",
"committed_date": "2019-05-03T12:48:33.000Z"
}
]
projects: [{
"id": 510,
"description": "",
"name": "tools.gitlabculture",
"name_with_namespace": "osp / tools.gitlabculture",
"path": "tools.gitlabculture",
"path_with_namespace": "osp/tools.gitlabculture",
"created_at": "2019-05-03T12:49:46.699Z",
"default_branch": "master",
"tag_list": [],
"ssh_url_to_repo": "git@gitlab.constantvzw.org:osp/tools.gitlabculture.git",
"http_url_to_repo": "https://gitlab.constantvzw.org/osp/tools.gitlabculture.git",
"web_url": "https://gitlab.constantvzw.org/osp/tools.gitlabculture",
"readme_url": "https://gitlab.constantvzw.org/osp/tools.gitlabculture/blob/master/README.md",
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-05-03T12:49:46.699Z",
"namespace": {
"id": 8,
"name": "osp",
"path": "osp",
"kind": "group",
"full_path": "osp",
"parent_id": null
},
"_links": {
"self": "https://gitlab.constantvzw.org/api/v4/projects/510",
"issues": "https://gitlab.constantvzw.org/api/v4/projects/510/issues",
"merge_requests": "https://gitlab.constantvzw.org/api/v4/projects/510/merge_requests",
"repo_branches": "https://gitlab.constantvzw.org/api/v4/projects/510/repository/branches",
"labels": "https://gitlab.constantvzw.org/api/v4/projects/510/labels",
"events": "https://gitlab.constantvzw.org/api/v4/projects/510/events",
"members": "https://gitlab.constantvzw.org/api/v4/projects/510/members"
},
"archived": false,
"visibility": "public",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 25,
"import_status": "none",
"open_issues_count": 0,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": null
}]
\ No newline at end of file
{{ project.name }}
<ul>
{% for file in project.tree %}
<li>{{ file.name }}</li>{% endfor %}
</ul>
<ul>
{% for commit in project.commits %}
<li>{{ commit.message }}</li>{% endfor %}
</ul>
\ No newline at end of file
<ul>
{% for project in projects %}
<li><a href="{{ project.links.self }}">{{ project.name }}</a></li>{% endfor %}
</ul>
\ No newline at end of file
def make_local_url(path):
return path
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment