issue #158: TRY to speed-up scope & $ref url-handling by keeping

fragments separated from URL (and avoid redunant frag/defrag).

Author: ankostis

jsonschema/compat.py

@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
-import sys
+
 import operator
+import sys
+
 
 try:
     from collections import MutableMapping, Sequence  # noqa
@@ -13,7 +15,8 @@
     zip = zip
     from io import StringIO
     from urllib.parse import (
-        unquote, urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit
+        unquote, urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit,
+        DefragResult
     )
     from urllib.request import urlopen
     str_types = str,
@@ -23,7 +26,8 @@
     from itertools import izip as zip  # noqa
     from StringIO import StringIO
     from urlparse import (
-        urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit # noqa
+        urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit, # noqa
+        DefragResult
     )
     from urllib import unquote  # noqa
     from urllib2 import urlopen  # noqa
@@ -47,7 +51,7 @@ def urldefrag(url):
     else:
         defrag = url
         frag = ''
-    return defrag, frag
+    return DefragResult(defrag, frag)
 
 
 # flake8: noqa

jsonschema/tests/test_benchmarks.py

@@ -45,6 +45,11 @@ def test_V3_meta_schema(self):
 
             278 runs in 2.00 sec
                 ms/run: mean(7.17), std(0.56), MIN(6.50), MAX(10.51)
+
+        splitted_fragments: Time @ Xeon 3.2GHz (x 1.05 faster)::
+
+            290 runs in 2.00 sec
+                ms/run: mean(6.88), std(0.66), MIN(6.00), MAX(12.00)
         """
 
         stats = []
@@ -65,6 +70,11 @@ def test_V4_meta_schema(self):
 
             164 runs in 2.00 sec
                 ms/run: mean(12.15), std(1.36), MIN(11.01), MAX(21.02)
+
+        splitted_fragments: Time @ Xeon 3.2GHz (x 1.16 faster)::
+
+            191 runs in 2.01 sec
+                ms/run: mean(10.47), std(0.81), MIN(9.50), MAX(14.00)
         """
 
         stats = []
@@ -85,6 +95,11 @@ def test_both_meta_schemas(self):
 
             104 runs in 2.01 sec
                 ms/run: mean(19.13), std(1.12), MIN(18.01), MAX(23.02)
+
+        splitted_fragments: Time @ Xeon 3.2GHz (x 1.11 faster)::
+
+            115 runs in 2.00 sec
+                ms/run: mean(17.24), std(1.24), MIN(16.00), MAX(24.51)
         """
 
         v_classes = [Draft3Validator, Draft4Validator]
@@ -107,6 +122,11 @@ def test_ref_model(self):
 
             16 runs in 2.00 sec
                 ms/run: mean(117.80), std(4.32), MIN(113.09), MAX(127.60)
+
+        splitted_fragments: Time @ Xeon 3.2GHz (x 1.12 faster)::
+
+            19 runs in 2.10 sec
+                ms/run: mean(105.07), std(2.45), MIN(102.02), MAX(109.02)
         """
 
         stats = []

jsonschema/validators.py

@@ -11,7 +11,8 @@
 
 from jsonschema import _utils, _validators
 from jsonschema.compat import (
-    Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen,
+    Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, DefragResult,
+
     str_types, int_types, iteritems,
 )
 from jsonschema.exceptions import ErrorTree  # Backwards compatibility  # noqa
@@ -222,7 +223,7 @@ class RefResolver(object):
 
     :argument str base_uri: URI of the referring document
     :argument referrer: the actual referring document
-    :argument dict store: a mapping from URIs to documents to cache
+    :argument dict store: a mapping from URIs (without fragments!) to documents to cache
     :argument bool cache_remote: whether remote refs should be cached after
         first resolution
     :argument dict handlers: a mapping from URI schemes to functions that
@@ -233,6 +234,7 @@ class RefResolver(object):
     def __init__(
         self, base_uri, referrer, store=(), cache_remote=True, handlers=(),
     ):
+        base_uri = urldefrag(base_uri)
         self.base_uri = base_uri
         self.resolution_scope = base_uri
         # This attribute is not used, it is for backwards compatibility
@@ -241,11 +243,11 @@ def __init__(
         self.handlers = dict(handlers)
 
         self.store = _utils.URIDict(
-            (id, validator.META_SCHEMA)
+            (id, validator.META_SCHEMA)         ## IDs assumed pure urls (no fragments).
             for id, validator in iteritems(meta_schemas)
         )
         self.store.update(store)
-        self.store[base_uri] = referrer
+        self.store[base_uri.url] = referrer
 
     @classmethod
     def from_schema(cls, schema, *args, **kwargs):
@@ -260,13 +262,22 @@ def from_schema(cls, schema, *args, **kwargs):
         return cls(schema.get(u"id", u""), schema, *args, **kwargs)
 
     @contextlib.contextmanager
-    def in_scope(self, scope):
-        old_scope = self.resolution_scope
-        self.resolution_scope = urljoin(old_scope, scope)
-        try:
+    def in_scope(self, scope, is_defragged=False):
+        if not scope:
             yield
-        finally:
-            self.resolution_scope = old_scope
+        else:
+            old_scope = self.resolution_scope
+            if not is_defragged:
+                scope = urldefrag(scope)
+            self.resolution_scope = DefragResult(
+                urljoin(old_scope.url, scope.url, allow_fragments=False)
+                        if scope.url else old_scope.url,
+                scope.fragment
+            )
+            try:
+                yield
+            finally:
+                self.resolution_scope = old_scope
 
     @contextlib.contextmanager
     def resolving(self, ref):
@@ -278,23 +289,24 @@ def resolving(self, ref):
 
         """
 
-        full_uri = urljoin(self.resolution_scope, ref)
-        uri, fragment = urldefrag(full_uri)
-        if not uri:
-            uri = self.base_uri
+        ref = urldefrag(ref)
 
-        if uri in self.store:
-            document = self.store[uri]
-        else:
+        url = urljoin(self.resolution_scope.url, ref.url, allow_fragments=False) \
+                if ref.url else self.resolution_scope.url
+
+        try:
+            document = self.store[url]
+        except KeyError:
             try:
-                document = self.resolve_remote(uri)
+                document = self.resolve_remote(url)
             except Exception as exc:
                 raise RefResolutionError(exc)
 
+        uri = DefragResult(url, ref.fragment)
         old_base_uri, self.base_uri = self.base_uri, uri
         try:
-            with self.in_scope(uri):
-                yield self.resolve_fragment(document, fragment)
+            with self.in_scope(uri, is_defragged=True):
+                yield self.resolve_fragment(document, ref.fragment)
         finally:
             self.base_uri = old_base_uri