[PATCH pglister] Add Archived-At header to delivered messages

This changeset adds a new header, Archived-At, to delivered messages.
This header field contains a direct link to the message in the web
archive. The Archived-At header is defined by RFC 5064:

  https://tools.ietf.org/html/rfc5064
---

Notes:
    To build the link, I used the urlpattern field from archive_server table
    and replace '/list/' by '/message-id/':
    
        https://www.postgresql.org/list/% -> https://www.postgresql.org/message-id/
    
    The urlpattern field is now retrieved in 'mailinglists' view, hence the
    migration.
    
    That's not very elegant, but I did not find a better way to achieve this.
    Perhaps adding a 'messageidpattern' column to 'lists_archiveserver' table
    would be better? Or maybe I missed something?
    
    Also, this is untested...

 lib/baselib/lists.py                          | 13 ++++--
 lib/handlers/mailhandler.py                   |  2 +-
 .../lists/migrations/0053_add_archivedat.py   | 41 +++++++++++++++++++
 3 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100644 web/pglister/lists/migrations/0053_add_archivedat.py

diff --git a/lib/baselib/lists.py b/lib/baselib/lists.py
index 76f8fc8..18bf4ae 100644
--- a/lib/baselib/lists.py
+++ b/lib/baselib/lists.py
@@ -1,4 +1,5 @@
 import re
+import urllib.parse
 import requests
 import psycopg2.extras
 
@@ -130,7 +131,7 @@ class ModerationReason(object):
 
 
 class MailingList(object):
-    def __init__(self, conn, id, address, name, domain, moderation_level, moderation_regex, spamscore_threshold,
maxsize,maxsizedrop, archive_server, archive_address, helppage, archive_domain, blocklist_regex, whitelist_regex,
tagged_delivery,tagkey, taglistsource, send_moderation_notices, cc_policy, bcc_policy, ignore_global_mod,
reject_to_cc):
+    def __init__(self, conn, id, address, name, domain, moderation_level, moderation_regex, spamscore_threshold,
maxsize,maxsizedrop, archive_server, archive_address, archive_urlpattern, helppage, archive_domain, blocklist_regex,
whitelist_regex,tagged_delivery, tagkey, taglistsource, send_moderation_notices, cc_policy, bcc_policy,
ignore_global_mod,reject_to_cc):
 
         """
         Internal, do not call directly! Instead use constructor methods:
         MailingList.get_by_address(address)
@@ -148,6 +149,7 @@ class MailingList(object):
         self.maxsizedrop = maxsizedrop
         self.archive_server = archive_server
         self.archive_address = archive_address
+        self.archive_urlpattern = archive_urlpattern
         self.helppage = helppage
         self.archive_domain = archive_domain
         self.blocklist_regexes = [re.compile(r, re.MULTILINE) for r in blocklist_regex.splitlines()]
@@ -171,7 +173,7 @@ class MailingList(object):
         if not maxsizedrop:
             self.maxsizedrop = int(config.get('defaults', 'size_drop'))
 
-    _SELECTFIELDS = "id, address, name, domain, moderation_level, moderation_regex, spamscore_threshold,
maxsizemoderate,maxsizedrop, archiveserver, archiveaddress, helppage, archivedomain, blocklist_regex, whitelist_regex,
tagged_delivery,tagkey, taglistsource, send_moderation_notices, cc_policy, bcc_policy, ignore_global_mod_regex,
reject_to_cc"
+    _SELECTFIELDS = "id, address, name, domain, moderation_level, moderation_regex, spamscore_threshold,
maxsizemoderate,maxsizedrop, archiveserver, archiveaddress, archiveurlpattern, helppage, archivedomain,
blocklist_regex,whitelist_regex, tagged_delivery, tagkey, taglistsource, send_moderation_notices, cc_policy,
bcc_policy,ignore_global_mod_regex, reject_to_cc"
 
 
     @staticmethod
     def get_by_address(conn, address):
@@ -279,7 +281,7 @@ OR EXISTS (SELECT 1 FROM mailinglist_whitelist WHERE listid=%(id)s AND email=%(e
         })
         return curs.fetchall()
 
-    def writeheaders(self, buf):
+    def writeheaders(self, buf, messageid):
         """
         Write the appropriate RFC2369 and RFC2919 headers to the
         buffer at the current location.
@@ -291,6 +293,7 @@ OR EXISTS (SELECT 1 FROM mailinglist_whitelist WHERE listid=%(id)s AND email=%(e
         buf.write("List-Owner: <mailto:{0}>\r\n".format(self.owner_address()).encode('utf8'))
         if self.archive_address:
             buf.write("List-Archive: <{0}>\r\n".format(self.archive_address).encode('utf8'))
+        buf.write("Archived-At: <{0}>\r\n".format(self.archived_at(messageid)).encode('utf-8'))
         buf.write("Precedence: bulk\r\n".encode('utf8'))
 
     def owner_address(self):
@@ -320,6 +323,10 @@ OR EXISTS (SELECT 1 FROM mailinglist_whitelist WHERE listid=%(id)s AND email=%(e
         else:
             return None
 
+    def archived_at(self, messageid):
+        msgid = urllib.parse.quote(messageid)
+        return self.archive_urlpattern.replace("/list/", "/message-id/").replace("%", msgid)
+
     def moderator_notice_address(self):
         """
         Return address used to send moderation notices. For now, we just
diff --git a/lib/handlers/mailhandler.py b/lib/handlers/mailhandler.py
index 863cfba..ae92cbb 100644
--- a/lib/handlers/mailhandler.py
+++ b/lib/handlers/mailhandler.py
@@ -759,7 +759,7 @@ ORDER BY 1""",
             if l.rstrip(b"\r\n") == b'':
                 # Empty line means we've hit the header boundary. Write
                 # our own custom headers and then switch to body mode.
-                self.mlist.writeheaders(targethdr)
+                self.mlist.writeheaders(targethdr, self.messageid)
                 break
             elif l == b'':
                 raise Exception("Reached end of input without finding end of headers")
diff --git a/web/pglister/lists/migrations/0053_add_archivedat.py
b/web/pglister/lists/migrations/0053_add_archivedat.py
new file mode 100644
index 0000000..4266e3b
--- /dev/null
+++ b/web/pglister/lists/migrations/0053_add_archivedat.py
@@ -0,0 +1,41 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lists', '0052_cannedunsubscriptionnotice'),
+    ]
+
+    operations = [
+        migrations.RunSQL("""
+CREATE OR REPLACE VIEW mailinglists AS
+   SELECT l.id AS id,
+         l.name || '@' || d.name AS address,
+      l.name AS name,
+      d.name AS domain,
+      l.moderation_level AS moderation_level,
+      l.moderation_regex AS moderation_regex,
+      l.spamscore_threshold AS spamscore_threshold,
+      l.maxsizemoderate AS maxsizemoderate,
+      l.maxsizedrop AS maxsizedrop,
+      a.name AS archiveserver,
+      replace(a.urlpattern, '%', l.name) AS archiveaddress,
+      a.urlpattern AS archiveurlpattern,
+      replace(d.lists_helppage, '%', l.name) AS helppage,
+      a.maildomain AS archivedomain,
+      l.blocklist_regex AS blocklist_regex,
+      l.whitelist_regex AS whitelist_regex,
+      l.tagged_delivery AS tagged_delivery,
+      l.tagkey AS tagkey,
+      l.taglistsource AS taglistsource,
+      l.send_moderation_notices AS send_moderation_notices,
+      l.cc_policy AS cc_policy,
+      l.bcc_policy AS bcc_policy,
+      l.ignore_global_mod_regex AS ignore_global_mod_regex,
+      l.reject_to_cc AS reject_to_cc
+   FROM lists_list l
+   INNER JOIN lists_domain d ON d.id=l.domain_id
+   LEFT JOIN lists_archiveserver a ON a.id=l.archivedat_id
+"""),
+    ]
-- 
2.20.1