Changeset 1639

Timestamp:

06/05/08 16:20:04 (6 months ago)

Author:

JensDiemer

Message:

Update IncludeRemote? Plugin:

• better charset handling
  
• add preformatted text mode
  

Location:

trunk/pylucid

Files:

• media/PyLucid/internal_page/IncludeRemote/IncludeRemote.html (modified) (1 diff)
• PyLucid/plugins_internal/IncludeRemote/IncludeRemote.py (modified) (5 diffs)
• PyLucid/plugins_internal/IncludeRemote/IncludeRemote_cfg.py (modified) (1 diff)

trunk/pylucid/media/PyLucid/internal_page/IncludeRemote/IncludeRemote.html

@@ -1 +1 @@
 <fieldset>
 {% if title %}<legend><a href="{{ url }}">{{ title }}</a></legend>{% endif %}
+{% if preformat %}<pre>{% endif %}
 {{ content }}
+{% if preformat %}</pre>{% endif %}
 </fieldset>
 <small class="IncludeRemote_info">

trunk/pylucid/PyLucid/plugins_internal/IncludeRemote/IncludeRemote.py

@@ -20 +20 @@
 
 
-import socket, urllib2, re, time
+import socket, urllib2, cgi, re, time
 
 socket.setdefaulttimeout(5) # set a timeout
@@ -29 +29 @@
 from PyLucid.system.BasePlugin import PyLucidBasePlugin
 
+STRIP_CONTENT = (
+    # stripe stylesheet links
+    re.compile('(<link.*?rel.*?stylesheet.*?>)(?is)'),
+    # strip javascript
+    re.compile("(<script.*?</script>)(?is)")
+)
+
+META_CHARSET = re.compile('<meta.*?charset=(.*?)"')
+BODY_RE = re.compile("<body.*?>(.*?)</body>(?is)")
 
 class IncludeRemote(PyLucidBasePlugin):
 
-    def lucidTag(self, url, title=None, escape=True):
-
+    def lucidTag(self, url, title=None, preformat=None, escape=True):
+        """
+        docu about the method args, look into config file!
+        """
         # get the remote content
         start_time = time.time()
@@ -40 +51 @@
             content = f.read()
             f.close()
-        except Exception, e:
-            return (
-                "<p>IncludeRemote error! Can't get '%s'<br />"
-                " error:'%s'</p>"
-            ) % (url, e)
+        except Exception, err:
+            if self.request.debug:
+                self.page_msg.red("Include remote '%s' error: %s" % (url, err))
+            return "[error getting '%s'.]" % title
+
         duration_time = time.time() - start_time
 
+        #______________________________________________________________________
+        # GET HEADER INFO
 
-        # cutout stylesheets
-        try:
-            content = re.sub(
-                '(<link.*?rel.*?stylesheet.*?>)(?is)',"",content
-            )
-        except:
-            pass
+        content_type = None
+        content_encodings = []
+
+        # detect content type and encoding
+        raw_content_type = f.headers.get("content-type")
+        if raw_content_type:
+            content_type, params = cgi.parse_header(raw_content_type)
+            if "charset" in params:
+                content_encodings.append(params["charset"])
 
 
-        # cutout JavaScripts
+        # Try to get content charset from html meta info
         try:
-            content = re.sub(
-                '(<script.*?</script>)(?is)',"",content
-            )
-        except:
+            charset = META_CHARSET.findall(content.lower())[0]
+        except IndexError:
             pass
-
+        except Exception, err:
+            if self.request.debug:
+                self.page_msg.red("Error get content charset:", err)
+        else:
+            content_encodings.append(charset)
 
         # decode into unicode
-        try:
-            charset = re.findall(
-                '<meta.*?content-type.*?charset=(.*?)"', content.lower()
-            )[0]
-            content = content.decode(charset)
-        except:
-            content = smart_unicode(content, errors='replace')
+        content = self._decode_content(content, content_encodings)
 
+        #______________________________________________________________________
 
         # try to cut out only the body content
         try:
-            content = re.findall("<body.*?>(.*?)</body>(?is)", content)[0]
-        except:
+            content = BODY_RE.findall(content)[0]
+        except IndexError:
             pass
+        except Exception, err:
+            if self.request.debug:
+                self.page_msg.red("Error strip body content:", err)
 
+        #______________________________________________________________________
+        # Strip content
+
+        for regex in STRIP_CONTENT:
+            try:
+                content = regex.sub(u"",content)
+            except Exception, err:
+                if self.request.debug:
+                    self.page_msg.red("Error strip content:", err)
+
+        #______________________________________________________________________
 
         if not escape:
@@ -87 +113 @@
             content = mark_safe(content)
 
+        # setup preformat
+        if preformat==None and "html" in content_type:
+            preformat = False
+        else:
+            preformat = True
 
         context = {
@@ -93 +124 @@
             "title": title,
             "content": content,
+            "preformat": preformat,
         }
         self._render_template("IncludeRemote", context)#, debug=True)
+
+    def _decode_content(self, content, content_encodings):
+        """
+        Try to decode content into unicode with the given encoding list.
+        """
+        if not content_encodings:
+            # No charset found.
+            return smart_unicode(content, errors='replace')
+
+        errors = []
+        for charset in content_encodings:
+            try:
+                content = content.decode(charset)
+            except UnicodeDecodeError, err:
+                errors.append(err)
+            else:
+                return content
+
+        if self.request.debug:
+            self.page_msg.red("Can't decode content:", errors)
+
+        return smart_unicode(content, errors='replace')

trunk/pylucid/PyLucid/plugins_internal/IncludeRemote/IncludeRemote_cfg.py

@@ -8 +8 @@
 __long_description__ = """
 Includes a remote website into your CMS page.
+
+=== parameters
+
+
+==== url:
+
+The url to get.
+
+
+==== title:
+
+A title for the fieldset header.
+
+
+==== preformat:
+
+Is the content preformatted text? Should we surround it with <pre> tags?
+
+preformat="None":
+    Automatic modus. If content type is html, don't surround with <pre> tag,
+    else add <pre> tag.
+
+preformat="True":
+    always add <pre> tag.
+
+preformat="False":
+    don't add <pre> tag
+
+
+==== escape:
+
+Is the receaved content safe (escape=False) or should we escape all html
+entries (escape=True)?
 """