Commit de32ea16 authored by gijs's avatar gijs
Browse files

A first version with cache invalidation.

parent 0c3d0ebf
......@@ -42,8 +42,18 @@ class ApiCall (object):
def invalidate_cache(self):
if self.has_cache:
debug('Invalidating cache `{}`'.format(self.cache_location))
os.unlink(self.cache_location)
def expire_cache_after (self, expiry_date):
if self.has_cache:
timestamp = os.path.getctime(self.cache_location)
# TODO: implement timezone-info
cache_date = datetime.datetime.fromtimestamp(timestamp)
if expiry_date > cache_date:
self.invalidate_cache()
def write_cache(self, data):
try:
with open(self.cache_location, 'w') as h:
......@@ -65,14 +75,7 @@ class ApiCall (object):
return self._cache
def get_api (self):
try:
_, data = self.get_api_url(self.url, self.query)
return data
except ApiError:
return None
def get_api_url(self, url, query = None):
def _make_api_call(self, url, query = None):
if query:
url = '{}?{}'.format(url, urlencode(query))
......@@ -87,10 +90,18 @@ class ApiCall (object):
# raise ApiError(url, e.code)
return None
def get (self):
if not self.has_cache:
data = self.get_api()
def read_from_api (self):
try:
_, data = self._make_api_call(self.url, self.query)
return data
except ApiError:
return None
def get (self, force_call=False):
if force_call or not self.has_cache:
data = self.read_from_api()
self.write_cache(data)
return self.read_cache()
def parse_api_result (self, raw):
......@@ -116,7 +127,7 @@ class ApiCallJson(ApiCall):
Returns values for the call. If the request is paginated go through
all pages
"""
def get_api(self):
def read_from_api(self):
query = self.query.copy() if self.query else {}
if self.paged:
......@@ -128,7 +139,7 @@ class ApiCallJson(ApiCall):
try:
while page and page < max_pages:
query['page'] = page
headers, page_data = self.get_api_url(self.url, query)
headers, page_data = self._make_api_call(self.url, query)
data.extend(page_data)
page = int(headers['X-Next-Page']) if headers['X-Next-Page'] else None
if page:
......@@ -137,7 +148,7 @@ class ApiCallJson(ApiCall):
return data
else:
query['per_page'] = 1
_, data = self.get_api_url(self.url, query)
_, data = self._make_api_call(self.url, query)
time.sleep(1/3)
return data
......
......@@ -2,9 +2,10 @@ import api
from jinja2 import Environment, FileSystemLoader, select_autoescape
import time
from models import Group
from settings import GROUP_ID, TEMPLATE_DIR, OUTPUT_DIR, CACHE_DIR
from settings import GROUP_ID, TEMPLATE_DIR, OUTPUT_DIR, CACHE_DIR, STATIC_DIR
import os.path
from os import makedirs, mkdir
import shutil
env = Environment(
loader=FileSystemLoader(TEMPLATE_DIR),
......@@ -20,6 +21,8 @@ if not os.path.exists(CACHE_DIR):
mkdir(CACHE_DIR)
group = Group(GROUP_ID)
# For now invalidate group cache on every call
group.invalidate_cache()
group.get()
# print(group)
# for project in group.projects:
......@@ -70,6 +73,11 @@ for project in group.projects:
project_template.stream(project=project).dump(os.path.join(output_path, project_name))
print('Copying static files')
if os.path.exists(os.path.join(OUTPUT_DIR, 'static')):
shutil.rmtree(os.path.join(OUTPUT_DIR, 'static'))
shutil.copytree(STATIC_DIR, os.path.join(OUTPUT_DIR, 'static'))
print('Generation finished')
# def update_project(project_id):
......
from utils import make_local_url
from utils import make_local_url, debug
from api import ApiCallJson, ApiCallJson, ApiCall, get_commits, get_group, get_projects, get_project, get_tree
import os.path
import markdown
import datetime
STRPTIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
"""
- Api Call encoded in the 'model'
......@@ -30,8 +33,8 @@ class Model (object):
def api_call (self):
raise NotImplementedError()
def get (self):
data = self.api_call.get()
def get (self, force_call=False):
data = self.api_call.get(force_call=force_call)
if data:
for k, v in data.items():
......@@ -82,6 +85,7 @@ class Project (Model):
# Derive the URL for the project from the name.
# Split on '.', remove last (filename)
# Remove first if it is OSP (old naming scheme)
# TODO: move this to a hook for wider compatibility?
parts = self.name.split('.')
path = parts[:-1]
name = '{}.html'.format(parts[-1])
......@@ -96,16 +100,30 @@ class Project (Model):
'self': make_local_url(os.path.join(*path, name))
}
@property
def last_activity_at (self):
if 'last_activity_at' in self.data:
return datetime.datetime.strptime(self.data['last_activity_at'], STRPTIME_FORMAT)
@property
def created_at (self):
if 'created_at' in self.data:
return datetime.datetime.strptime(self.data['created_at'], STRPTIME_FORMAT)
@property
def commits (self):
if not hasattr(self, '_commits'):
self._commits = Commits(get_commits(project_id=self.id).get())
call = get_commits(project_id=self.id)
call.expire_cache_after(self.last_activity_at)
self._commits = Commits(call.get())
return self._commits
@property
def tree (self):
if not hasattr(self, '_tree'):
self._tree = Tree(get_tree(project_id=self.id).get())
call = get_tree(project_id=self.id)
call.expire_cache_after()
self._tree = Tree(call.get())
return self._tree
@property
......@@ -114,8 +132,8 @@ class Project (Model):
pass
@property
def api_url (self):
return get_project(self.id).get()
def api_call (self):
return get_project(self.id)
class Projects (Collection):
model = Project
......@@ -132,6 +150,9 @@ class Group (Model):
return self._projects
def invalidate_cache(self):
self.api_call.invalidate_cache()
get_projects(self.id).invalidate_cache()
class Commits (Collection):
model = Commit
......
......@@ -4,6 +4,7 @@ BASE_DIR = os.path.dirname(__file__)
CACHE_DIR = os.path.join(BASE_DIR, 'cache')
TEMPLATE_DIR = os.path.join(BASE_DIR, 'templates')
OUTPUT_DIR = os.path.join(BASE_DIR, 'output')
STATIC_DIR = os.path.join(BASE_DIR, 'static')
API_URL = "https://gitlab.constantvzw.org/api/v4/"
GROUP_ID = 8
......
......@@ -34,7 +34,7 @@
</main>
<script src="http://localhost:8000/static/js/pad.js" type="text/javascript" charset="utf-8"></script>
<script src="/static/js/pad.js" type="text/javascript" charset="utf-8"></script>
{% block script %}
{% endblock %}
......
from settings import DEBUG
def make_local_url(path):
return path
\ No newline at end of file
return path
def debug (msg):
if DEBUG:
print(msg)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment