Changeset 1639 for trunk/pylucid/PyLucid/plugins_internal/IncludeRemote/IncludeRemote.py

Timestamp:

06/05/08 16:20:04 (22 months ago)

Author:

JensDiemer

Message:

Update IncludeRemote? Plugin:

• better charset handling
• add preformatted text mode

Files:

• trunk/pylucid/PyLucid/plugins_internal/IncludeRemote/IncludeRemote.py (modified) (5 diffs)

trunk/pylucid/PyLucid/plugins_internal/IncludeRemote/IncludeRemote.py

@@ -20 +20 @@
 
 
-import socket, urllib2, re, time
+import socket, urllib2, cgi, re, time
 
 socket.setdefaulttimeout(5) # set a timeout
@@ -29 +29 @@
 from PyLucid.system.BasePlugin import PyLucidBasePlugin
 
+STRIP_CONTENT = (
+    # stripe stylesheet links
+    re.compile('(<link.*?rel.*?stylesheet.*?>)(?is)'),
+    # strip javascript
+    re.compile("(<script.*?</script>)(?is)")
+)
+
+META_CHARSET = re.compile('<meta.*?charset=(.*?)"')
+BODY_RE = re.compile("<body.*?>(.*?)</body>(?is)")
 
 class IncludeRemote(PyLucidBasePlugin):
 
-    def lucidTag(self, url, title=None, escape=True):
-
+    def lucidTag(self, url, title=None, preformat=None, escape=True):
+        """
+        docu about the method args, look into config file!
+        """
         # get the remote content
         start_time = time.time()
@@ -40 +51 @@
             content = f.read()
             f.close()
-        except Exception, e:
-            return (
-                "<p>IncludeRemote error! Can't get '%s'<br />"
-                " error:'%s'</p>"
-            ) % (url, e)
+        except Exception, err:
+            if self.request.debug:
+                self.page_msg.red("Include remote '%s' error: %s" % (url, err))
+            return "[error getting '%s'.]" % title
+
         duration_time = time.time() - start_time
 
+        #______________________________________________________________________
+        # GET HEADER INFO
 
-        # cutout stylesheets
-        try:
-            content = re.sub(
-                '(<link.*?rel.*?stylesheet.*?>)(?is)',"",content
-            )
-        except:
-            pass
+        content_type = None
+        content_encodings = []
+
+        # detect content type and encoding
+        raw_content_type = f.headers.get("content-type")
+        if raw_content_type:
+            content_type, params = cgi.parse_header(raw_content_type)
+            if "charset" in params:
+                content_encodings.append(params["charset"])
 
 
-        # cutout JavaScripts
+        # Try to get content charset from html meta info
         try:
-            content = re.sub(
-                '(<script.*?</script>)(?is)',"",content
-            )
-        except:
+            charset = META_CHARSET.findall(content.lower())[0]
+        except IndexError:
             pass
-
+        except Exception, err:
+            if self.request.debug:
+                self.page_msg.red("Error get content charset:", err)
+        else:
+            content_encodings.append(charset)
 
         # decode into unicode
-        try:
-            charset = re.findall(
-                '<meta.*?content-type.*?charset=(.*?)"', content.lower()
-            )[0]
-            content = content.decode(charset)
-        except:
-            content = smart_unicode(content, errors='replace')
+        content = self._decode_content(content, content_encodings)
 
+        #______________________________________________________________________
 
         # try to cut out only the body content
         try:
-            content = re.findall("<body.*?>(.*?)</body>(?is)", content)[0]
-        except:
+            content = BODY_RE.findall(content)[0]
+        except IndexError:
             pass
+        except Exception, err:
+            if self.request.debug:
+                self.page_msg.red("Error strip body content:", err)
 
+        #______________________________________________________________________
+        # Strip content
+
+        for regex in STRIP_CONTENT:
+            try:
+                content = regex.sub(u"",content)
+            except Exception, err:
+                if self.request.debug:
+                    self.page_msg.red("Error strip content:", err)
+
+        #______________________________________________________________________
 
         if not escape:
@@ -87 +113 @@
             content = mark_safe(content)
 
+        # setup preformat
+        if preformat==None and "html" in content_type:
+            preformat = False
+        else:
+            preformat = True
 
         context = {
@@ -93 +124 @@
             "title": title,
             "content": content,
+            "preformat": preformat,
         }
         self._render_template("IncludeRemote", context)#, debug=True)
+
+    def _decode_content(self, content, content_encodings):
+        """
+        Try to decode content into unicode with the given encoding list.
+        """
+        if not content_encodings:
+            # No charset found.
+            return smart_unicode(content, errors='replace')
+
+        errors = []
+        for charset in content_encodings:
+            try:
+                content = content.decode(charset)
+            except UnicodeDecodeError, err:
+                errors.append(err)
+            else:
+                return content
+
+        if self.request.debug:
+            self.page_msg.red("Can't decode content:", errors)
+
+        return smart_unicode(content, errors='replace')