Changeset 1510

Timestamp:

03/27/08 16:59:44 (7 months ago)

Author:

JensDiemer

Message:

add a experimental new cache middleware
-merge pagestats into cache middleware
-add unitest for the cache
-move setup_debug() from the views into the common middleware
-update other tests

Location:

trunk/pylucid

Files:

• PyLucid/db/__init__.py (modified) (1 diff)
• PyLucid/index.py (modified) (7 diffs)
• PyLucid/install/tests.py (modified) (3 diffs)
• PyLucid/middlewares/cache.py (added)
• PyLucid/middlewares/common.py (modified) (3 diffs)
• PyLucid/middlewares/pagestats.py (deleted)
• PyLucid/models.py (modified) (5 diffs)
• PyLucid/settings_example.py (modified) (2 diffs)
• PyLucid/template_addons/filters.py (modified) (1 diff)
• tests/cache.py (added)
• tests/internal_pages.py (modified) (1 diff)
• tests/plugin_backlinks.py (modified) (2 diffs)
• tests/plugin_main_menu.py (modified) (2 diffs)
• tests/stylesheet.py (modified) (3 diffs)
• tests/utils/BrowserDebug.py (modified) (1 diff)
• tests/__init__.py (modified) (6 diffs)

trunk/pylucid/PyLucid/db/__init__.py

@@ -1 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-
-"""
-shared DB Objects.
-"""
-
-import page

trunk/pylucid/PyLucid/index.py

@@ -18 +18 @@
 """
 
-import datetime, md5
-
 from django.http import HttpResponse, HttpResponsePermanentRedirect, \
                                                            HttpResponseRedirect
+from django.conf import settings
 from django.template import RequestContext
-from django.core.cache import cache
+from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext as _
-from django.utils.safestring import mark_safe
-from django.conf import settings
-
-from PyLucid import models
-
+
+from PyLucid.models import Page
 from PyLucid.system import plugin_manager
+from PyLucid.system.URLs import URLs
+from PyLucid.system.page_msg import PageMessages
 from PyLucid.system.response import SimpleStringIO
 from PyLucid.system.exceptions import AccessDenied
-from PyLucid.system.page_msg import PageMessages
+from PyLucid.system.context_processors import add_dynamic_context, add_css_tag
 from PyLucid.system.detect_page import get_current_page_obj, \
                                                             get_default_page_id
-from PyLucid.system.URLs import URLs
-from PyLucid.system.context_processors import add_dynamic_context, add_css_tag
-from PyLucid.system.utils import setup_debug
+from PyLucid.tools.utils import escape, escape_django_tags
 from PyLucid.tools.content_processors import apply_markup, \
                                     render_string_template, redirect_warnings
-from PyLucid.tools.utils import escape, escape_django_tags
 from PyLucid.plugins_internal.page_style.page_style import replace_add_data
 
@@ -135 +130 @@
 
 
-def patch_response_headers(response, cache_timeout, ETag, last_modified):
-    """
-    Adds some useful headers to the given HttpResponse object:
-        ETag, Last-Modified, Expires and Cache-Control
-
-    Original version: django.utils.cache.patch_response_headers()
-    """
-    response['ETag'] = ETag
-    response['Last-Modified'] = last_modified.strftime(
-        '%a, %d %b %Y %H:%M:%S GMT'
-    )
-    now = datetime.datetime.utcnow()
-    expires = now + datetime.timedelta(0, cache_timeout)
-    response['Expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S GMT')
-
-
-def get_cached_data(url):
-    """
-    -Build the cache_key from the given url. Use the last page shortcut.
-    -retuned the cache_key and the page data.
-    """
-    if url == "":
-        # Request without a shortcut -> request the default page
-        shortcut = "/"
-    else:
-        # Note: We use append_slash, but the url pattern striped the last
-        #     slash out.
-        # e.g.: '/page1/page2/page3' -> ['/page1/page2', 'page3'] -> 'page3'
-        shortcut = url.rsplit("/", 1)[-1]
-
-    cache_key = settings.PAGE_CACHE_PREFIX + shortcut
-    #print "Used cache key:", cache_key
-
-    # Get the page data from the cache.
-    response = cache.get(cache_key)
-
-    return cache_key, response
-
-
 def index(request, url):
     """
@@ -181 +137 @@
     the page shortcut from the url.
     """
-    # Cache only for anonymous users. Otherwise users how are log-in don't see
-    # the dynamic integrate admin menu.
-    use_cache = request.user.is_anonymous()
-
-    if use_cache:
-        # Try to get the cms page request from the cache
-        cache_key, response = get_cached_data(url)
-        if response:
-            # This page has been cached in the past, use the cache data:
-            return response
-
-    setup_debug(request)
-
     try:
         current_page_obj = get_current_page_obj(request, url)
@@ -211 +154 @@
 
     context = _get_context(request, current_page_obj)
+
     # Get the response for the requested cms page:
     response = _render_cms_page(request, context)
 
-    if use_cache:
-        # It's a anonymous user -> Cache the cms page.
-        cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
-        # Add some headers for the browser cache
-        patch_response_headers(
-            response, cache_timeout,
-            ETag = md5.new(cache_key).hexdigest(),
-            last_modified = current_page_obj.lastupdatetime,
-        )
-        # Save the page into the cache
-        cache.set(cache_key, response, cache_timeout)
+    if getattr(request, "_use_cache", None) == None:
+        # Set _use_cache information for the PyLucid cache middleware, but only
+        # if it was set to true or false in the past
+        request._use_cache = True
 
     return response
+
 
 def _get_page(request, page_id):
@@ -233 +171 @@
     TODO: Check int(page_id)!
     """
-    setup_debug(request)
-
     try:
-        current_page_obj = models.Page.objects.get(id=int(page_id))
-    except models.Page.DoesNotExist:
+        current_page_obj = Page.objects.get(id=int(page_id))
+    except Page.DoesNotExist:
         # The ID in the url is wrong -> goto the default page
         default_page_id = get_default_page_id()
-        current_page_obj = models.Page.objects.get(id=default_page_id)
+        current_page_obj = Page.objects.get(id=default_page_id)
 
         user = request.user
@@ -256 +192 @@
 
     return current_page_obj
+
 
 def handle_command(request, page_id, module_name, method_name, url_args):
@@ -329 +266 @@
     return HttpResponsePermanentRedirect(url)
 
+
 def permalink(request, page_id):
     """

trunk/pylucid/PyLucid/install/tests.py

@@ -4 +4 @@
 """
 
+import os, cgi, sys, time
+
+from django.conf import settings
+
+from django import newforms as forms
+from django.core import management
+from django.core.cache import cache
+
 from PyLucid.install.BaseInstall import BaseInstall
 from PyLucid.system.response import SimpleStringIO
-
-from django.core import management
-from django import newforms as forms
-
-import os, cgi, sys, time
 
 
@@ -22 +25 @@
         print txt
         print "-"*80
+
+    def _test_cache(self):
+        """
+        Test the cache backend.
+        """
+        if settings.CACHE_BACKEND.startswith("dummy"):
+            return
+        self._print_infoline(
+            "Test cache backend '%s':" % settings.CACHE_BACKEND
+        )
+        cache_key = "cache test"
+        content = "A cache test content..."
+        cache_timeout = 50
+        print "Save into cache with key '%s'." % cache_key
+        cache.set(cache_key, content, cache_timeout)
+        print "Try to get the saved content from the cache."
+        cached_content = cache.get(cache_key)
+        if cached_content == None:
+            print " * Get None back. Cache didn't work!"
+            return
+        elif cached_content==content:
+            print " * Cache works fine ;)"
+        else:
+            # Should never appears
+            print " * Error! Cache content not the same!"
+
+        print "Try to delete the cache entry."
+        cache.delete(cache_key)
+        cached_content = cache.get(cache_key)
+        if cached_content == None:
+            print "OK, entry deleted."
+        else:
+            print "Error: entry not deleted!"
+
 
     def _verbose_try_import(self, module_name):
@@ -109 +146 @@
 
     def print_info(self):
+        self._test_cache()
+
         self._print_infoline("Tests for the memcache backend:")
         self._verbose_try_import("cmemcache")

trunk/pylucid/PyLucid/middlewares/common.py

@@ -31 +31 @@
 from PyLucid.system.exceptions import LowLevelError
 from PyLucid.system.template import render_help_page
+from PyLucid.system.utils import setup_debug
 
 
@@ -58 +59 @@
     """
     def process_request(self, request):
+        # add the attribute "debug" to the request object.
+        setup_debug(request)
+
         try:
             session_middleware.process_request(request)
@@ -64 +68 @@
         except Exception, e:
             raise_non_table_error(e)
-
 
     def process_view(self, request, view_func, view_args, view_kwargs):

trunk/pylucid/PyLucid/models.py

@@ -18 +18 @@
 import os, posixpath, pickle
 
+from django.conf import settings
 from django.db import models
-from django.contrib.auth.models import User, Group
-from django.contrib.auth.models import UNUSABLE_PASSWORD
+from django.core.cache import cache
+from django.contrib.auth.models import User, Group, UNUSABLE_PASSWORD
 from django.utils.translation import ugettext as _
-from django.conf import settings
 
 from PyLucid.tools.shortcuts import getUniqueShortcut
 from PyLucid.tools import crypt
 from PyLucid.system.utils import get_uri_base
+#from PyLucid.db.cache import delete_page_cache
 
 
@@ -38 +39 @@
 )
 
+def delete_page_cache():
+    """
+    Delete all pages in the cache.
+    Needed, if:
+        - A template has been edited
+        - The menu changes (edit the page name, position, parent link)
+    TODO: move this function from models.py into a other nice place...
+    """
+    for items in Page.objects.values('shortcut').iterator():
+        shortcut = items["shortcut"]
+        cache_key = settings.PAGE_CACHE_PREFIX + shortcut
+        cache.delete(cache_key)
+
 
 class Page(models.Model):
     """
     A CMS Page Object
+
+    TODO: We should refactor the "pre_save" behavior, use signals:
+    http://code.djangoproject.com/wiki/Signals
     """
     # Explicite id field, so we can insert a help_text ;)
@@ -239 +256 @@
             self._check_parent(self.id)
 
+        # Delete all pages in the cache.
+        # FIXME: This is only needed, if the menu changed: e.g.: if the page
+        # position, shortcut, parent cahnges...
+        delete_page_cache()
+
         # Rebuild shortcut / make shortcut unique:
         self._prepare_shortcut()
+
+        # Delete old page cache, if exist
+        cache_key = settings.PAGE_CACHE_PREFIX + self.shortcut
+        cache.delete(cache_key)
 
         super(Page, self).save() # Call the "real" save() method
@@ -711 +737 @@
             pass
 
+        #Delete the page cache if a stylesheet was edited.
+        delete_page_cache()
+
         super(Style, self).save() # Call the "real" save() method
 
@@ -729 +758 @@
     description = models.TextField()
     content = models.TextField()
+
+    def save(self):
+        """
+        Delete the page cache if a template was edited.
+        """
+        delete_page_cache()
+
+        super(Template, self).save() # Call the "real" save() method
 
     class Admin:

trunk/pylucid/PyLucid/settings_example.py

@@ -90 +90 @@
     'PyLucid.middlewares.common.PyLucidCommonMiddleware',
 
+    'PyLucid.middlewares.cache.CacheMiddleware',
+
     'django.middleware.common.CommonMiddleware',
     'django.middleware.doc.XViewMiddleware',
@@ -95 +97 @@
     # Add a human readable anchor to every html headline:
     'PyLucid.middlewares.headline_anchor.HeadlineAnchor',
-
-    # Insert a statistic line into the generated page:
-    'PyLucid.middlewares.pagestats.PageStatsMiddleware',
 )
 

trunk/pylucid/PyLucid/template_addons/filters.py

@@ -58 +58 @@
     """
     Converts a time duration into a friendly text representation.
-    Note: Used in PageStatsMiddleware, too.
+    Note: Used in the PyLucid cache middleware, too.
     """
     if t<1:

trunk/pylucid/tests/internal_pages.py

@@ -54 +54 @@
         Page.objects.all().delete() # Delete all existins pages
 
-        self.template = tests.create_template(TEST_TEMPLATE)
+        self.template = tests.create_template(content = TEST_TEMPLATE)
 
         # Create the test pages defined in content_test_utils.py

trunk/pylucid/tests/plugin_backlinks.py

@@ -41 +41 @@
         Page.objects.all().delete() # Delete all existins pages
 
-        self.template = tests.create_template("{% lucidTag back_links %}")
+        self.template = tests.create_template(
+            content = "{% lucidTag back_links %}"
+        )
 
         # Create the test pages defined in content_test_utils.py
@@ -74 +76 @@
         print_last_page="True"
         """
-
         self.template.content = (
             '{% lucidTag back_links print_last_page="True" %}'

trunk/pylucid/tests/plugin_main_menu.py

@@ -36 +36 @@
         Page.objects.all().delete() # Delete all existins pages
 
-        self.template = tests.create_template("{% lucidTag main_menu %}")
+        self.template = tests.create_template(
+            content = "{% lucidTag main_menu %}"
+        )
 
         tests.create_pages(tests.TEST_PAGES, template=self.template)
@@ -97 +99 @@
         Page.objects.all().delete() # Delete all existins pages
 
-        self.template = tests.create_template("{% lucidTag main_menu %}")
+        self.template = tests.create_template(
+            content = "{% lucidTag main_menu %}"
+        )
 
     def test_base(self):

trunk/pylucid/tests/stylesheet.py

@@ -52 +52 @@
         Page.objects.all().delete() # Delete all existins pages
 
-        self.template = tests.create_template("{% lucidTag page_style %}")
+        self.template = tests.create_template(
+            content = "{% lucidTag page_style %}"
+        )
         self.style = tests.create_stylesheet(
             name = "TestStyle", content = TEST_STYLE_CONTENT,
@@ -90 +92 @@
         """ Return the stylesheet link contains in the root cms page. """
         response = self.client.get("/")
+        from_cache = response.get("from_cache", None)
+        print from_cache
         content = response.content
         links = self._exctract_stylelinks(content)
@@ -150 +154 @@
         must_link = self._get_stylelink()
         is_link = self._get_content_link()
-        assert is_link == must_link
+        self.failUnlessEqual(must_link, is_link)
 
     def test_command_link(self):

trunk/pylucid/tests/utils/BrowserDebug.py

@@ -58 +58 @@
     stack_info = "".join(stack)
 
-    info = (
-        "\n<br /><hr />\n"
-        "<h3>Unittest info</h3>\n"
-        "<dl>\n"
-        "<dt>url:</dt><dd>%s</dd>\n"
-        "<dt>traceback:</dt><dd><pre>%s</pre></dd>\n"
-        "</dl>\n"
-        "</body>"
-    ) % (url, stack_info)
-
-    content = content.replace("</body>", info)
+    if "</body>" in content:
+        info = (
+            "\n<br /><hr />\n"
+            "<h3>Unittest info</h3>\n"
+            "<dl>\n"
+            "<dt>url:</dt><dd>%s</dd>\n"
+            "<dt>response info:</dt><dd><pre>%s</pre></dd>\n"
+            "<dt>traceback:</dt><dd><pre>%s</pre></dd>\n"
+            "</dl>\n"
+            "</body>"
+        ) % (url, response, stack_info)
+        content = content.replace("</body>", info)
+    else:
+        # Not a html page?
+        content += "\n<pre>\n"
+        content += "-" * 79
+        content += (
+            "\nUnittest info\n"
+            "=============\n"
+            "url: %s\n"
+            "response info:\n%s\n"
+            "traceback:\n%s\n</pre>"
+        ) % (url, response, stack_info)
 
 

trunk/pylucid/tests/__init__.py

@@ -225 +225 @@
             is_list2 = pprint.pformat(is_list)
             should_be_list2 = pprint.pformat(should_be_list)
+            print "\nThe two lists are not the same:"
             for line in make_diff(should_be_list2, is_list2):
                 print line
@@ -313 +314 @@
 
     # _________________________________________________________________________
-    # common tests:
+    # link snapshot:
+
     def link_snapshot_test(self, snapshot):
         """
@@ -330 +332 @@
 
         self.assertLists(is_links, should_be_links, sort=False)
+
+    def create_link_snapshot(self, print_result=True):
+        """
+        Build a a reference snapshot for a unittest.
+        Display it via pprint and returned it, too.
+        Usefull for copy&paste the output into this source file :)
+        """
+        if print_result:
+            print "Build a snapshot for the unittest compare:"
+            print "-"*79
+        data = {}
+        for page in Page.objects.all():
+            url = page.get_absolute_url()
+
+            response = self.client.get(url)
+            self.failUnlessEqual(response.status_code, 200)
+
+            content = response.content.strip()
+            links = self.get_links(content)
+
+            data[url] = links
+
+        if print_result:
+            pprint.pprint(data)
+            print "-"*79
+        return data
 
 
@@ -391 +419 @@
 
 
-def create_template(content):
+def create_template(**kwargs):
     """
     Delete all existing templates, create a new one and return the instance.
@@ -397 +425 @@
     Template.objects.all().delete()
 
-    template = Template(content = content)
+    template = Template(**kwargs)
     template.save()
     return template
@@ -421 +449 @@
     default_template = Template.objects.all()[0]
     default_style = Style.objects.all()[0]
-    default_markup = 0 # html withdout TinyMCE
+    default_markup = 0 # html without TinyMCE
 
     page = Page(
         name             = data.get("name", "New Page"),
         shortcut         = data.get("shortcut", None),
+        content          = data.get("content", "TestPageContent"),
         template         = data.get("template", default_template),
         style            = data.get("style", default_style),