Merge branch 'pad-management' of gitlab.constantvzw.org:osp/tools.ethertoff into pad-management

ethertoff/templatetags/__init__.py

-1
\ No newline at end of file
+

ethertoff/urls.py

-from django.contrib.auth import views as auth_views
 from django.urls import path, re_path
 from . import views
 
-
-# Uncomment the next two lines to enable the admin:
-from django.contrib import admin
-admin.autodiscover()
+## FIXME: if uncommenting url namespacing will be needed
+# app_name = "ethertoff"
 
 urlpatterns = [
     path('', views.home, name='home'),
-    path('admin/', admin.site.urls),
-    path('all/', views.all, name='all'), # TemplateView.as_view(template_name = 'all.html')
-    #re_path(r'manage/(?P<page>[\d]+)?/?$', views.manage, name='manage'),
-    path('manage', views.manage, name='manage'),
+    path('all/', views.all, name='all'),
     path('manage/', views.manage, name='manage'),
-    path('manage/<path:path>', views.manage, name='manage'),
+    path('manage/<path:path>/', views.manage, name='manage'),
     path('publish/', views.publish, name='publish'),
     path('generate/', views.generate, name='generate'),
     path('css-screen/', views.css, name='css-screen'),
     path('css-print/', views.cssprint, name='css-print'),
     path('css-offset/', views.offsetprint, name='css-offset'),
     path('css-slide/', views.css_slide, name='css-slide'),
-    re_path(r'^(?P<slug>[^/]+)\.xhtml$', views.xhtml, name='xhtml'),
-    path('accounts/login', auth_views.LoginView.as_view(),
-        {'template_name': 'login.html'}, name='login'),
-    path('accounts/logout', auth_views.LogoutView.as_view(),
-        {'template_name': 'logout.html'}, name='logout'),
-    path('accounts/password_change', auth_views.PasswordChangeView.as_view(), name="password_change"),
-    path('accounts/password_change_done', auth_views.PasswordChangeDoneView.as_view(), name="password_change_done"),
     path('create/', views.padCreate, name='pad-create'),
-    path('create/<path:prefix>', views.padCreate, name='pad-create'),
-    re_path(r'^rename/(?P<pk>\d+)/$', views.padRename, name='pad-rename'),
-    re_path(r'^delete/(?P<pk>\d+)/$', views.padDelete, name='pad-delete'),
-    re_path(r'^public/(?P<pk>\d+)/$', views.padPublic, name='pad-public'),
-    re_path(r'^private/(?P<pk>\d+)/$', views.padPrivate, name='pad-private'),
+    path('create/<path:prefix>/', views.padCreate, name='pad-create'),
+    path('rename/<int:pk>/', views.padRename, name='pad-rename'),
+    path('delete/<int:pk>/', views.padDelete, name='pad-delete'),
+    path('public/<int:pk>/', views.padPublic, name='pad-public'),
+    path('private/<int:pk>/', views.padPrivate, name='pad-private'),
     re_path(r'(?P<mode>[r|s|p])/(?P<slug>[^/]+)$', views.pad_read, name='pad-read'),
     re_path(r'(?P<mode>[r|s|p])/(?P<slug>[^/]+)$', views.pad_read, name='pad'),
     re_path(r'(?P<mode>[w])/(?P<slug>[^/]+)$', views.pad, name='pad'),
-    #re_path(r'r/(?P<slug>[^/]+)$', views.pad, name='pad-read'),
-    #re_path(r's/(?P<slug>[^/]+)$', views.pad, name='pad-slide'),
-    #re_path(r'p/(?P<slug>[^/]+)$', views.pad, name='pad-print'),
-    re_path(r'w/(?P<slug>[^/]+)$', views.pad, name='pad-write')
+    re_path(r'w/(?P<slug>[^/]+)$', views.pad, name='pad-write'),
+    re_path(r'^(?P<slug>[^/]+)\.xhtml$', views.xhtml, name='xhtml'),
 ]

ethertoff/views.py

@@ -596,7 +596,6 @@ def pad_read(request, mode="r", slug=None):
 
         meta_list = list(meta.items())
 
-        print(meta_list)
 
     tpl_params = { 'pad'                : pad,
                    'meta'               : meta,      # to access by hash, like meta.author

ethertoff_rdf_index/admin.py

+from django.contrib import admin
+
+# Register your models here.

ethertoff_rdf_index/apps.py

+from django.apps import AppConfig
+
+
+class EthertoffRdfindexConfig(AppConfig):
+    name = 'ethertoff_rdfindex'

ethertoff_rdf_index/management/commands/rdf_index.py

+from django.core.management.base import BaseCommand
+from etherpadlite.models import Pad
+from django.contrib.sites.models import Site
+from django.urls import reverse
+from aasniff import AAApp
+import markdown
+from markdown.extensions.toc import TocExtension
+from django.test import Client
+import html5lib
+from rdflib.plugins.memory import IOMemory
+import rdflib
+from ... import settings as app_settings
+
+
+def tidy(string):
+    parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("dom"))
+    dom = parser.parse(string)
+
+    # FIXME: remove this? we moved to rdflib
+    # Redland crashes if no xmlns attribute is declared.
+    # see: http://bugs.librdf.org/mantis/view.php?id=521
+    # Lets fix it in the meanwhile...
+    elt = dom.getElementsByTagName("html")[0]
+    if not elt.hasAttribute("xmlns"):
+        elt.setAttribute("xmlns", "http://www.w3.org/1999/xhtml")
+
+    return dom.toxml()
+
+
+class Conf(object):
+    SNIFFERS = [
+        'HttpSniffer',
+        'HtmlSniffer',
+    ]
+
+    STORE = app_settings.STORE 
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Indexes pages'
+
+    def handle(self, *args, **options):
+        from django.contrib.sites.models import Site
+        domain = Site.objects.get_current().domain
+
+        app = AAApp(conf=Conf)
+
+        # store = IOMemory()
+        # graph = rdflib.graph.ConjunctiveGraph(store=store)
+
+        for pad in Pad.objects.filter():
+            c = Client()
+            path = reverse('pad-read', kwargs={'mode': 'r', 'slug': pad.display_slug})
+            response = c.get(path)
+            url = f"http://{domain}{path}"
+            
+            print(f"parsing {url}")
+            if response.status_code == 200:
+                try:
+                    app.graph.parse(data=tidy(response.content), format="rdfa", publicID=url)
+                except:
+                    print(f"couldn't parse {url}")
+            else:
+                print(f"failed to parse {url}")
+
+
+        # for quad in graph.quads():
+        #     # avoids duplicates statements
+        #     app.graph.remove(quad)
+        #     # adds the new statements
+        #     app.graph.add(quad)

ethertoff_rdf_index/management/commands/rdf_show.py

+from django.core.management.base import BaseCommand
+from etherpadlite.models import Pad
+from django.contrib.sites.models import Site
+from django.urls import reverse
+from aasniff import AAApp
+import rdflib
+from ... import settings as app_settings
+
+
+class Conf(object):
+    SNIFFERS = [
+        'HttpSniffer',
+        'HtmlSniffer',
+    ]
+
+    STORE = app_settings.STORE 
+
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Print indexed data and texts'
+
+    def handle(self, *args, **options):
+        app = AAApp(conf=Conf)
+
+        for quad in app.graph.quads():
+            print(quad)
+
+        node = rdflib.URIRef("http://purl.org/dc/terms/title")
+        NS = {
+            'bibo': rdflib.Namespace("http://purl.org/ontology/bibo/"),
+            'bib': rdflib.Namespace("http://purl.org/net/biblio#"),
+            'dc': rdflib.namespace.DC,
+            'dcterms': rdflib.namespace.DCTERMS,
+            'egr2': rdflib.Namespace("http://rdvocab.info/ElementsGr2/"),
+            'foaf': rdflib.namespace.FOAF,
+            'frbr': rdflib.Namespace("http://purl.org/vocab/frbr/core#"),
+            'purl': rdflib.Namespace("http://purl.org/dc/terms/"),
+            'rdf': rdflib.RDF,
+            'rdvocab': rdflib.Namespace("http://RDVocab.info/elements/"),
+            'stats': rdflib.Namespace("http://kavan.land/vocab/stats#"),
+            'vocab': rdflib.Namespace("http://purl.org/vocab/frbr/core#"),
+            'wemi': rdflib.Namespace("http://RDVocab.info/RDARelationshipsWEMI/"),
+            'z': rdflib.Namespace("http://www.zotero.org/namespaces/export#"),
+        }
+
+        as_predicate = app.graph.query("""
+            SELECT DISTINCT ?subject ?title
+            WHERE {
+                ?subject dcterms:title ?title.
+            }
+        """, initBindings={'predicate': node}, initNs=NS)
+
+        for i in as_predicate:
+            print(i[1])
+
+

ethertoff_rdf_index/models.py

+from django.db import models
+
+# Create your models here.

ethertoff_rdf_index/settings.py

+from django.conf import settings
+
+# OPTION_C = getattr(settings, '_'.join([NAMESPACE, 'OPTION_C']), None)
+# if OPTION_C is None:
+#     raise ImproperlyConfigured('...')
+STORE = getattr(settings, "ETHERTOFF_RDF_STORE", {
+    'ENGINE': 'sqlite',
+    'NAME': 'aasniff.sqlite',
+})

ethertoff_rdf_index/tests.py

+from django.test import TestCase
+
+# Create your tests here.

ethertoff_rdf_index/views.py

+from django.shortcuts import render
+
+# Create your views here.

my_project/local_settings.py.example

@@ -35,3 +35,4 @@ MEDIA_ROOT = ''
 # Example: "/var/www/example.com/static/"
 STATIC_ROOT = '/home/osp/static-ethertoff-collected/'
 
+EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

my_project/settings.py

@@ -33,6 +33,7 @@ ALLOWED_HOSTS = []
 INSTALLED_APPS = [
     # 'example_theme', ## Add your theme here if any
     'ethertoff',
+    'ethertoff_rdf_index',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',

my_project/urls.py

@@ -19,9 +19,9 @@ try:
 except AttributeError:
     pass
 
-app_name = "ethertoff"
-
 urlpatterns = [
+    path('admin/', admin.site.urls),
+    path('accounts/', include('django.contrib.auth.urls')),
     path('', include("ethertoff.urls")),
 ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)