diff --git a/etherdump/commands/dump.py b/etherdump/commands/dump.py
index bd1815e8ed2ef85a6f3975c6c5a842f22551fe00..6e345badde9e53835b5c7f1f6d5a0d6a37e8dddb 100644
--- a/etherdump/commands/dump.py
+++ b/etherdump/commands/dump.py
@@ -1,6 +1,11 @@
#!/usr/bin/env python
# License: AGPL
#
+#
+# todo:
+# Capture exceptions... add HTTP status errors (502) to meta!!
+# so that an eventual index can show the problematic pages!
+# Also: provide links to text only / html versions when diff HTML fails
from __future__ import print_function
from etherdump import DATAPATH
@@ -12,6 +17,7 @@ from datetime import datetime
from xml.etree import cElementTree as ET
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError
+from time import sleep
# external dependencies (use pip to install these)
import html5lib, jinja2
@@ -102,6 +108,11 @@ def get_template_env (tpath=None):
env = jinja2.Environment(loader=loader)
return env
+def get_group_info(gid, info):
+ if 'groups' in info:
+ if gid in info['groups']:
+ return info['groups'][gid]
+
def main(args):
p = ArgumentParser("""
_ _ _
@@ -117,7 +128,6 @@ def main(args):
p.add_argument("--verbose", default=False, action="store_true", help="flag for verbose output")
p.add_argument("--limit", type=int, default=None)
p.add_argument("--allpads", default=False, action="store_true", help="flag to process all pads")
- p.add_argument("--spider", default=False, action="store_true", help="flag to spider pads")
p.add_argument("--templatepath", default=os.path.join(DATAPATH, "templates"), help="directory with templates (override default files)")
p.add_argument("--colors-template", default="pad_colors.html", help="pad with authorship colors template name: pad_colors.html")
p.add_argument("--padlink", default=[], action="append", help="give a pad link pattern, example: 'http\:\/\/10\.1\.10\.1/p/(.*)'")
@@ -126,8 +136,11 @@ def main(args):
p.add_argument("--showurls", default=False, action="store_true", help="flag to display API URLs that are used (to stderr)")
p.add_argument("--hidepaths", default=False, action="store_true", help="flag to not display paths")
p.add_argument("--pretend", default=False, action="store_true", help="flag to not actually save")
+ p.add_argument("--linkify", default=False, action="store_true", help="flag to process [[link]] forms (and follow when --spider is used)")
+ p.add_argument("--spider", default=False, action="store_true", help="flag to spider pads (requires --linkify)")
p.add_argument("--add-images", default=False, action="store_true", help="flag to add image tags")
- p.add_argument("--authors-css", default="authors.css", help="filename to save collected authorship css (nb: etherdump will overwrite this file!)")
+ p.add_argument("--force", default=False, action="store_true", help="force dump (even if not updated since last dump)")
+ p.add_argument("--authors-css", default=None, help="filename to save collected authorship css (nb: any existing file will be mercilessly overwritten), default: don't accumulate css")
# TODO css from pad --- ie specify a padid for a stylesheet!!!!!!
# p.add_argument("--css", default="styles.css", help="padid of stylesheet")
@@ -175,205 +188,270 @@ def main(args):
done.add(padid)
data['padID'] = padid.encode("utf-8")
+
if args.verbose:
- print ("PADID \"{0}\"".format(padid).encode("utf-8"), file=sys.stderr)
+ print (u"PADID \"{0}\"".format(padid).encode("utf-8"), file=sys.stderr)
+
+ # g.yIRLMysh0PMsCMHc$
+ grouppat = re.compile(ur"^g\.(\w+)\$(.+)$")
+ m = grouppat.search(padid)
+ if m:
+ group = m.group(1)
+ ginfo = get_group_info(group, info)
+ if not ginfo:
+ print ("No info for group '{0}', skipping".format(group), file=sys.stderr)
+ continue
+ padid = m.group(2)
+ else:
+ group = None
+ ginfo = None
+
if not args.pretend:
try:
- os.makedirs(args.path)
+ if ginfo:
+ os.makedirs(os.path.join(args.path, ginfo['name']))
+ else:
+ os.makedirs(args.path)
except OSError:
pass
- try:
-
- # _
- # _ __ ___ ___| |_ __ _
- # | '_ ` _ \ / _ \ __/ _` |
- # | | | | | | __/ || (_| |
- # |_| |_| |_|\___|\__\__,_|
-
- meta_url = urlify(padid, ext=".json")
- meta_out = "{0}/{1}".format(args.path, meta_url.encode("utf-8"))
- raw_url = urlify(padid, ext=".txt")
- raw_out = "{0}/{1}".format(args.path, raw_url.encode("utf-8"))
- colors_url = urlify(padid, ext=".html")
- colors_out = "{0}/{1}".format(args.path, colors_url.encode("utf-8"))
-
- if not args.hidepaths:
- print (meta_out, file=sys.stderr)
- if not args.pretend:
- meta = {}
- meta['padid'] = padid
- revisions_url = apiurl+'getRevisionsCount?'+urlencode(data)
- if args.showurls:
- print (revisions_url, file=sys.stderr)
- meta['total_revisions'] = json.load(urlopen(revisions_url))['data']['revisions']
+ retry = True
+ tries = 1
+ while retry:
+ retry = False
+ try:
- lastedited_url = apiurl+'getLastEdited?'+urlencode(data)
- if args.showurls:
- print (lastedited_url, file=sys.stderr)
- lastedited_raw = json.load(urlopen(lastedited_url))['data']['lastEdited']
- meta['lastedited_raw'] = lastedited_raw
- meta['lastedited'] = datetime.fromtimestamp(int(lastedited_raw)/1000).isoformat()
+ # _
+ # _ __ ___ ___| |_ __ _
+ # | '_ ` _ \ / _ \ __/ _` |
+ # | | | | | | __/ || (_| |
+ # |_| |_| |_|\___|\__\__,_|
- # author_ids (unfortunately, this is a list of internal etherpad author ids -- not the names ppl type)
- authors_url = apiurl+'listAuthorsOfPad?'+urlencode(data)
- if args.showurls:
- print (authors_url, file=sys.stderr)
- meta['author_ids'] = json.load(urlopen(authors_url))['data']['authorIDs']
- meta['colors'] = colors_url
- meta['raw'] = raw_url
- meta['meta'] = meta_url
- with open(meta_out, "w") as f:
- json.dump(meta, f)
-
- # _ __ __ ___ __
- # | '__/ _` \ \ /\ / /
- # | | | (_| |\ V V /
- # |_| \__,_| \_/\_/
-
- if not args.hidepaths:
- print (raw_out, file=sys.stderr)
- text_url = apiurl+"getText?"+urlencode(data)
- if args.showurls:
- print (text_url, file=sys.stderr)
- if not args.pretend:
- rawText = json.load(urlopen(text_url))['data']['text']
- with open(raw_out, "w") as f:
- f.write(rawText.encode("utf-8"))
-
- # _ _ _
- # | |__ | |_ _ __ ___ | |
- # | '_ \| __| '_ ` _ \| |
- # | | | | |_| | | | | | |
- # |_| |_|\__|_| |_| |_|_|
-
- # todo ? -- regular HTML output
-
- # _
- # ___ ___ | | ___ _ __ ___
- # / __/ _ \| |/ _ \| '__/ __|
- # | (_| (_) | | (_) | | \__ \
- # \___\___/|_|\___/|_| |___/
-
- if not args.hidepaths:
- print (colors_out, file=sys.stderr)
- data['startRev'] = "0"
- colors_url = apiurl+'createDiffHTML?'+urlencode(data)
- if args.showurls:
- print (colors_url, file=sys.stderr)
- html = json.load(urlopen(colors_url))['data']['html']
- t = html5lib.parse(html, namespaceHTMLElements=False)
- trim_removed_spans(t)
- html = ET.tostring(t, method="html")
-
- # Stage 1: Process as text
- # Process [[wikilink]] style links
- # and (optionally) add linked page names to spider todo list
- html, links = linkify(html)
- if args.spider:
- for l in links:
- if l not in todo and l not in done:
- if l.startswith("http://") or l.startswith("https://"):
+ meta_url = urlify(padid, ext=".json")
+ raw_url = urlify(padid, ext=".txt")
+ colors_url = urlify(padid, ext=".html")
+
+ if ginfo:
+ meta_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], meta_url.encode("utf-8"))
+ raw_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], raw_url.encode("utf-8"))
+ colors_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], colors_url.encode("utf-8"))
+ else:
+ meta_out = "{0}/{1}".format(args.path, meta_url.encode("utf-8"))
+ raw_out = "{0}/{1}".format(args.path, raw_url.encode("utf-8"))
+ colors_out = "{0}/{1}".format(args.path, colors_url.encode("utf-8"))
+
+ if not args.pretend:
+ meta = {}
+ meta['padid'] = padid
+ revisions_url = apiurl+'getRevisionsCount?'+urlencode(data)
+ if args.showurls:
+ print (revisions_url, file=sys.stderr)
+ meta['total_revisions'] = json.load(urlopen(revisions_url))['data']['revisions']
+
+ # CHECK REVISIONS (against existing meta)
+ if meta['total_revisions'] == 0:
+ if args.verbose:
+ print (" pad has no revisions, skipping", file=sys.stderr)
+ continue
+ if os.path.exists(meta_out):
+ with open(meta_out) as f:
+ old_meta = json.load(f)
+ if not args.force and old_meta['total_revisions'] == meta['total_revisions']:
if args.verbose:
- print ("Ignoring absolute URL in [[ link ]] form", file=sys.stderr)
+ print (" skipping (up to date)", file=sys.stderr)
continue
- # if args.verbose:
- # print (" link: {0}".format(l), file=sys.stderr)
- todo.append(l)
-
- # Stage 2: Process as ElementTree
- #
- t = html5lib.parse(html, namespaceHTMLElements=False)
- # apply linkpats
- for a in t.findall(".//a"):
- href = a.attrib.get("href")
- original_href = href
- if href:
- # if args.verbose:
- # print ("searching for PADLINK: {0}".format(href))
- for pat in padlinkpats:
- if re.search(pat, href) != None:
- # if args.verbose:
- # print (" found PADLINK: {0}".format(href))
- href = re.sub(pat, "\\1.html", href)
- padid = filename_to_padid(href)
- set_text_contents(a, "[[{0}]]".format(padid))
- if padid not in todo and padid not in done:
- if args.verbose:
- print (" link: {0}".format(padid), file=sys.stderr)
- todo.append(padid)
- # apply linkpats
- for s, r in linkpats:
- href = re.sub(s, r, href)
- if href != original_href:
- old_contents = text_contents(a)
- # print ("OLD_CONTENTS {0}".format(old_contents))
- if old_contents == original_href:
- if args.verbose:
- print (" Updating href IN TEXT", file=sys.stderr)
- set_text_contents(a, href)
-
- if original_href != href:
+
+ lastedited_url = apiurl+'getLastEdited?'+urlencode(data)
+ if args.showurls:
+ print (lastedited_url, file=sys.stderr)
+ lastedited_raw = json.load(urlopen(lastedited_url))['data']['lastEdited']
+ meta['lastedited_raw'] = lastedited_raw
+ meta['lastedited'] = datetime.fromtimestamp(int(lastedited_raw)/1000).isoformat()
+
+ # author_ids (unfortunately, this is a list of internal etherpad author ids -- not the names ppl type)
+ authors_url = apiurl+'listAuthorsOfPad?'+urlencode(data)
+ if args.showurls:
+ print (authors_url, file=sys.stderr)
+ meta['author_ids'] = json.load(urlopen(authors_url))['data']['authorIDs']
+ meta['colors'] = colors_url
+ meta['raw'] = raw_url
+ meta['meta'] = meta_url
+ # defer output to LAST STEP (as confirmation)
+
+ # _ __ __ ___ __
+ # | '__/ _` \ \ /\ / /
+ # | | | (_| |\ V V /
+ # |_| \__,_| \_/\_/
+
+ text_url = apiurl+"getText?"+urlencode(data)
+ if args.showurls:
+ print (text_url, file=sys.stderr)
+ if not args.pretend:
+ rawText = json.load(urlopen(text_url))['data']['text']
+ if rawText.strip() == "":
if args.verbose:
- print (" Changed href from {0} to {1}".format(original_href, href), file=sys.stderr)
- a.attrib['href'] = href
-
- # SHOWIMAGES : inject img tag for (local) images
- if args.add_images:
- ext = os.path.splitext(href)[1].lower().lstrip(".")
- if ext in ("png", "gif", "jpeg", "jpg"):
- # ap = _parent(a)
- print ("Adding img '{0}'".format(href), file=sys.stderr)
- img = ET.SubElement(a, "img")
- br = ET.SubElement(a, "br")
- a.remove(img); a.insert(0, img)
- a.remove(br); a.insert(1, br)
- img.attrib['src'] = href
-
- # extract the style tag (with authorship colors)
- style = t.find(".//style")
- if style != None:
- if args.authors_css:
- for i in style.text.splitlines():
- if len(i):
- selector, rule = i.split(' ',1)
- authors_css_rules[selector] = rule
- style = '' # strip the individual style tag from each page (only exports to authors-css file)
- # nb: it's up to the template to refer to the authors-css file
+ print (" empty text, skipping", file=sys.stderr)
+ continue
+ if not args.hidepaths:
+ print (raw_out, file=sys.stderr)
+ with open(raw_out, "w") as f:
+ f.write(rawText.encode("utf-8"))
+
+ # _ _ _
+ # | |__ | |_ _ __ ___ | |
+ # | '_ \| __| '_ ` _ \| |
+ # | | | | |_| | | | | | |
+ # |_| |_|\__|_| |_| |_|_|
+
+ # todo ? -- regular HTML output
+
+ # _
+ # ___ ___ | | ___ _ __ ___
+ # / __/ _ \| |/ _ \| '__/ __|
+ # | (_| (_) | | (_) | | \__ \
+ # \___\___/|_|\___/|_| |___/
+
+ if not args.hidepaths:
+ print (colors_out, file=sys.stderr)
+ data['startRev'] = "0"
+ colors_url = apiurl+'createDiffHTML?'+urlencode(data)
+ if args.showurls:
+ print (colors_url, file=sys.stderr)
+ html = json.load(urlopen(colors_url))['data']['html']
+ t = html5lib.parse(html, namespaceHTMLElements=False)
+ trim_removed_spans(t)
+ html = ET.tostring(t, method="html")
+
+ # Stage 1: Process as text
+ # Process [[wikilink]] style links
+ # and (optionally) add linked page names to spider todo list
+ if args.linkify:
+ html, links = linkify(html)
+ if args.spider:
+ for l in links:
+ if l not in todo and l not in done:
+ if l.startswith("http://") or l.startswith("https://"):
+ if args.verbose:
+ print ("Ignoring absolute URL in [[ link ]] form", file=sys.stderr)
+ continue
+ # if args.verbose:
+ # print (" link: {0}".format(l), file=sys.stderr)
+ todo.append(l)
+
+ # Stage 2: Process as ElementTree
+ #
+ t = html5lib.parse(html, namespaceHTMLElements=False)
+ # apply linkpats
+ for a in t.findall(".//a"):
+ href = a.attrib.get("href")
+ original_href = href
+ if href:
+ # if args.verbose:
+ # print ("searching for PADLINK: {0}".format(href))
+ for pat in padlinkpats:
+ if re.search(pat, href) != None:
+ # if args.verbose:
+ # print (" found PADLINK: {0}".format(href))
+ href = re.sub(pat, "\\1.html", href)
+ padid = filename_to_padid(href)
+ set_text_contents(a, "[[{0}]]".format(padid))
+ if padid not in todo and padid not in done:
+ if args.verbose:
+ print (" link: {0}".format(padid), file=sys.stderr)
+ todo.append(padid)
+ # apply linkpats
+ for s, r in linkpats:
+ href = re.sub(s, r, href)
+ if href != original_href:
+ old_contents = text_contents(a)
+ # print ("OLD_CONTENTS {0}".format(old_contents))
+ if old_contents == original_href:
+ if args.verbose:
+ print (" Updating href IN TEXT", file=sys.stderr)
+ set_text_contents(a, href)
+
+ if original_href != href:
+ if args.verbose:
+ print (" Changed href from {0} to {1}".format(original_href, href), file=sys.stderr)
+ a.attrib['href'] = href
+
+ # SHOWIMAGES : inject img tag for (local) images
+ if args.add_images:
+ ext = os.path.splitext(href)[1].lower().lstrip(".")
+ if ext in ("png", "gif", "jpeg", "jpg"):
+ # ap = _parent(a)
+ print ("Adding img '{0}'".format(href), file=sys.stderr)
+ img = ET.SubElement(a, "img")
+ br = ET.SubElement(a, "br")
+ a.remove(img); a.insert(0, img)
+ a.remove(br); a.insert(1, br)
+ img.attrib['src'] = href
+
+ # extract the style tag (with authorship colors)
+ style = t.find(".//style")
+ if style != None:
+ if args.authors_css:
+ for i in style.text.splitlines():
+ if len(i):
+ selector, rule = i.split(' ',1)
+ authors_css_rules[selector] = rule
+ # replace individual style with a ref to the authors-css
+ style = '<link rel="stylesheet" type="text/css" href="{0}">'.format(args.authors_css)
+ else:
+ style = ET.tostring(style, method="html")
else:
- style = ET.tostring(style, method="html")
- else:
- style = ""
- # and extract the contents of the body
- html = contents(t.find(".//body"))
-
- if not args.pretend:
- with open(colors_out, "w") as f:
- # f.write(html.encode("utf-8"))
- f.write(colors_template.render(
- html = html,
- style = style,
- revision = meta['total_revisions'],
- padid = padid,
- timestamp = datetime.now(),
- meta_url = meta_url,
- raw_url = raw_url,
- colors_url = colors_url,
- lastedited = meta['lastedited']
- ).encode("utf-8"))
-
- # _
- # | | ___ ___ _ __
- # | |/ _ \ / _ \| '_ \
- # | | (_) | (_) | |_) |
- # |_|\___/ \___/| .__/
- # |_|
-
- count += 1
- if args.limit and count >= args.limit:
- break
- except TypeError:
- print ("ERROR, skipping!", file=sys.stderr)
+ style = ""
+ # and extract the contents of the body
+ html = contents(t.find(".//body"))
+
+ if not args.pretend:
+ with open(colors_out, "w") as f:
+ # f.write(html.encode("utf-8"))
+ f.write(colors_template.render(
+ html = html,
+ style = style,
+ revision = meta['total_revisions'],
+ padid = padid,
+ timestamp = datetime.now(),
+ meta_url = meta_url,
+ raw_url = raw_url,
+ colors_url = colors_url,
+ lastedited = meta['lastedited']
+ ).encode("utf-8"))
+
+ # OUTPUT METADATA (finally)
+ if not args.hidepaths:
+ print (meta_out, file=sys.stderr)
+ with open(meta_out, "w") as f:
+ json.dump(meta, f)
+ # _
+ # | | ___ ___ _ __
+ # | |/ _ \ / _ \| '_ \
+ # | | (_) | (_) | |_) |
+ # |_|\___/ \___/| .__/
+ # |_|
+
+ count += 1
+ if args.limit and count >= args.limit:
+ break
+
+ # except HTTPError as e:
+ # retry = True
+
+ # except TypeError as e:
+ # print ("TypeError, skipping!", file=sys.stderr)
+
+ except Exception as e:
+ print ("[{0}] Exception: {1}".format(tries, e), file=sys.stderr)
+ sleep(3)
+ retry = True
+
+ if retry:
+ tries += 1
+ if tries > 5:
+ print (" GIVING UP", file=sys.stderr)
+ retry = False
+
# Write the unified CSS with authors
if args.authors_css:
diff --git a/etherdump/data/templates/pad_colors.html b/etherdump/data/templates/pad_colors.html
index b74d93fd6468092c46445194381af5dfd561104a..39cdf25938b359ea583d51c8ae1aca4a39f235be 100644
--- a/etherdump/data/templates/pad_colors.html
+++ b/etherdump/data/templates/pad_colors.html
@@ -5,7 +5,6 @@
<meta charset="utf-8">
<meta revision="{{revision}}">
<link rel="stylesheet" type="text/css" href="pad.css">
-<link rel="stylesheet" type="text/css" href="authors.css">
{{ style }}
</head>
<body>
diff --git a/padinfo.sample.json b/padinfo.sample.json
index ea53748174bf627667494834fb7dd9bd0d728de8..128a333f0fd4fcfdbd65b2947966b9fdf1b59c2b 100644
--- a/padinfo.sample.json
+++ b/padinfo.sample.json
@@ -4,5 +4,9 @@
"hostname": "localhost",
"apiversion": "1.2.9",
"apiurl": "/api/",
- "apikey": "8f55f9ede1b3f5d88b3c54eb638225a7bb71c64867786b608abacfdb7d418be1"
+ "apikey": "8f55f9ede1b3f5d88b3c54eb638225a7bb71c64867786b608abacfdb7d418be1",
+ "groups": {
+ "71FpVh4MZBvl8VZ6": {"name": "Transmediale", "id": 43},
+ "HyYfoX3Q6S5utxs5": {"name": "test", "id": 42 }
+ }
}