First working version of the generator

generator/.gitignore

+*.pyc
+__pycache__
+output/*
+cache/*
\ No newline at end of file

generator/api.py

+from urllib.request import URLError, urlopen
+from urllib.parse import urlencode, urljoin
+import os.path
+import os
+import json
+import datetime
+
+CACHE_DIR = os.path.join(os.path.dirname(__file__), 'cache')
+API_URL = "https://gitlab.constantvzw.org/api/v4/"
+GROUP_ID = 8
+
+DEBUG = True
+MAX_PAGES = 100
+PER_PAGE = 100
+
+def debug (msg):
+  if DEBUG:
+    print(msg)
+
+class ApiError(Exception):
+    def __init__(self, url, what):
+        self.url = url
+        self.what = what
+    def __str__(self):
+        return '(%s) => %s'%(self.url,self.what)
+
+class ApiCall (object):
+  def __init__ (self, api_path, api_url = API_URL, cache_dir = CACHE_DIR):
+    self._cache = None
+    self.api_path = list(map(str, api_path))
+    self.api_url = api_url
+    self.cache_dir = cache_dir
+
+  @property
+  def cache_file(self):
+    return os.path.join(self.cache_dir, '{}.json'.format('.'.join(self.api_path)))
+
+  @property
+  def url(self):
+    return urljoin(self.api_url, '/'.join(self.api_path))
+
+  @property
+  def has_cache(self):
+    return os.path.exists(self.cache_file)
+
+  def invalidate_cache(self):
+    if self.has_cache:
+      os.unlink(self.cache_file)
+
+  def make_cache(self):
+    try:
+      obj = self.get_api()
+      json.dump(obj, open(self.cache_file, 'w'))
+    except ApiError as e:
+      json.dump({
+        'reason': e.what,
+        'timestamp': datetime.datetime.now().timestamp()               
+      }, open(self.cache_file, 'w'))
+      obj = []
+
+    self._cache = obj
+
+  def load_cache(self):
+    debug('Hit cache {}'.format(self.url))
+    if not self._cache:
+      obj = json.load(open(self.cache_file, 'r'))
+
+      if 'reason' in obj:
+        self._cache = []
+      else:
+        self._cache = obj
+
+    return self._cache
+
+  """
+    Returns values for the call. If the request is paginated go through
+    all pages
+  """
+  def get_api(self):
+    page = 1
+    items = []
+    while page < MAX_PAGES:
+      headers, pageitems = self.get_api_page(page)
+      items.extend(pageitems)
+      if page >= int(headers['X-Total-Pages']):
+        return items
+      else:
+        page += 1
+
+  def get_api_page(self, page=0):
+    debug('{}, page {}'.format(self.url, page))
+
+    try:
+      q = urlencode({'page': page, 'per_page': PER_PAGE})
+      url = '{}?{}'.format(self.url, q)
+      res = urlopen(url)
+      return ({ k: v for (k,v) in res.getheaders() }, json.loads(res.read()))
+    except URLError as e:
+      if hasattr(e, 'reason'):
+        raise ApiError(url, e.reason)
+      elif hasattr(e, 'code'):
+        raise ApiError(url, e.code)
+    return None
+
+  def get (self):
+    if self.has_cache:
+      return self.load_cache()
+    else:
+      self.make_cache()
+      return self.load_cache()
+    
+# A way to make a call, cache and be able te remove the cache when needed
+# prepare a call, set of url and local cache file
+def group_projects (group = GROUP_ID):
+  return ApiCall(['groups', group, 'projects'])
+
+def commits (project_id = None):
+  if project_id is not None:
+    return ApiCall(['projects', project_id, 'repository', 'commits'])
+
+def tree (project_id = None):
+  if project_id is not None:
+    return ApiCall(['projects', project_id, 'repository', 'tree'])

generator/generate.py

+import api
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+import time
+from models import Project
+
+env = Environment(
+    loader=FileSystemLoader('templates'),
+    autoescape=select_autoescape(['html', 'xml'])
+)
+
+
+def get_projects ():
+  call = api.group_projects()
+  return call.get()
+
+
+def get_tree (project_id):
+  call = api.tree(project_id)
+  return call.get()
+
+
+def get_commits (project_id):
+  call = api.commits(project_id)
+  return call.get()
+
+
+def make_local_url (url):
+  return url
+
+
+def parse_project (project):
+  return Project(id=project['id'], tree = get_tree(project['id']), commits = get_commits(project['id']))
+
+print('Loading templates')
+projects = [parse_project(project) for project in get_projects()]
+
+print('Starting generation')
+template = env.get_template('projects.html')
+template.stream(projects=projects).dump('output/index.html') 
+
+project_template = env.get_template('project.html')
+
+for project in projects:
+  project_template.stream(project=project).dump('output/{}.html'.format(project.name))
+
+print('Generation finished')
+
+def update_project(project_id):
+  call = get_commits(project_id)
+  call.invalidate_cache()
+  call.get()
\ No newline at end of file

generator/models.py

+import utils
+
+class Project (object):
+  def __init__ (self, id=None, name=None, tree=[], readme=None, commits = []):
+    self.id = id,
+    self.name = name,
+    self.tree = tree,
+    self.readme = readme,
+    self.commits = commits
+
+  @property
+  def links (self):
+    return {
+      'self': utils.make_local_url('{}.html'.format(self.name))
+    }
\ No newline at end of file

generator/readme.md

+// deal with api error
+// count attempts ? perhaps record timestamp
+// 404 → cache
+
+{
+  'error': 404,
+  'timestamp': x
+}
+
+
+
+tree: [  {
+    "id": "591330eb757b8f3da46258f89b530bdac7c92e17",
+    "name": "contour.html",
+    "type": "blob",
+    "path": "contour.html",
+    "mode": "100644"
+  }
+]
+
+commits: [  {
+    "id": "e14c0a584be4dc086fc48938b848a16003cd22ea",
+    "short_id": "e14c0a58",
+    "created_at": "2019-05-03T12:48:33.000Z",
+    "parent_ids": [],
+    "title": "A new project: the remake of OSP website!",
+    "message": "A new project: the remake of OSP website!\n",
+    "author_name": "Stephanie Vilayphiou",
+    "author_email": "stephanie@stdin.fr",
+    "authored_date": "2019-05-03T12:48:33.000Z",
+    "committer_name": "Stephanie Vilayphiou",
+    "committer_email": "stephanie@stdin.fr",
+    "committed_date": "2019-05-03T12:48:33.000Z"
+  }
+]
+
+projects: [{
+  "id": 510,
+  "description": "",
+  "name": "tools.gitlabculture",
+  "name_with_namespace": "osp / tools.gitlabculture",
+  "path": "tools.gitlabculture",
+  "path_with_namespace": "osp/tools.gitlabculture",
+  "created_at": "2019-05-03T12:49:46.699Z",
+  "default_branch": "master",
+  "tag_list": [],
+  "ssh_url_to_repo": "git@gitlab.constantvzw.org:osp/tools.gitlabculture.git",
+  "http_url_to_repo": "https://gitlab.constantvzw.org/osp/tools.gitlabculture.git",
+  "web_url": "https://gitlab.constantvzw.org/osp/tools.gitlabculture",
+  "readme_url": "https://gitlab.constantvzw.org/osp/tools.gitlabculture/blob/master/README.md",
+  "avatar_url": null,
+  "star_count": 0,
+  "forks_count": 0,
+  "last_activity_at": "2019-05-03T12:49:46.699Z",
+  "namespace": {
+    "id": 8,
+    "name": "osp",
+    "path": "osp",
+    "kind": "group",
+    "full_path": "osp",
+    "parent_id": null
+  },
+  "_links": {
+    "self": "https://gitlab.constantvzw.org/api/v4/projects/510",
+    "issues": "https://gitlab.constantvzw.org/api/v4/projects/510/issues",
+    "merge_requests": "https://gitlab.constantvzw.org/api/v4/projects/510/merge_requests",
+    "repo_branches": "https://gitlab.constantvzw.org/api/v4/projects/510/repository/branches",
+    "labels": "https://gitlab.constantvzw.org/api/v4/projects/510/labels",
+    "events": "https://gitlab.constantvzw.org/api/v4/projects/510/events",
+    "members": "https://gitlab.constantvzw.org/api/v4/projects/510/members"
+  },
+  "archived": false,
+  "visibility": "public",
+  "resolve_outdated_diff_discussions": false,
+  "container_registry_enabled": true,
+  "issues_enabled": true,
+  "merge_requests_enabled": true,
+  "wiki_enabled": true,
+  "jobs_enabled": true,
+  "snippets_enabled": true,
+  "shared_runners_enabled": true,
+  "lfs_enabled": true,
+  "creator_id": 25,
+  "import_status": "none",
+  "open_issues_count": 0,
+  "public_jobs": true,
+  "ci_config_path": null,
+  "shared_with_groups": [],
+  "only_allow_merge_if_pipeline_succeeds": false,
+  "request_access_enabled": false,
+  "only_allow_merge_if_all_discussions_are_resolved": false,
+  "printing_merge_request_link_enabled": true,
+  "merge_method": "merge",
+  "external_authorization_classification_label": null
+}]
\ No newline at end of file

generator/templates/project.html

+{{ project.name }}
+<ul>
+  {% for file in project.tree %}
+    <li>{{ file.name }}</li>{% endfor %}
+</ul>
+
+<ul>
+  {% for commit in project.commits %}
+    <li>{{ commit.message }}</li>{% endfor %}
+</ul>
\ No newline at end of file

generator/templates/projects.html

+<ul>
+  {% for project in projects %}
+    <li><a href="{{ project.links.self }}">{{ project.name }}</a></li>{% endfor %}
+</ul>
\ No newline at end of file

generator/utils.py

+def make_local_url(path):
+  return path
\ No newline at end of file