issue python-jsonschema#158: TRY to speed-up scope & $ref url-handling by keeping

fragments separated from URL (and avoid redunant frag/defrag).

Author: ankostis

jsonschema/compat.py

@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
-import sys
+
 import operator
+import sys
+
 
 try:
     from collections import MutableMapping, Sequence  # noqa
@@ -13,7 +15,8 @@
     zip = zip
     from io import StringIO
     from urllib.parse import (
-        unquote, urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit
+        unquote, urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit,
+        DefragResult
     )
     from urllib.request import urlopen
     str_types = str,
@@ -23,7 +26,8 @@
     from itertools import izip as zip  # noqa
     from StringIO import StringIO
     from urlparse import (
-        urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit # noqa
+        urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit, # noqa
+        DefragResult
     )
     from urllib import unquote  # noqa
     from urllib2 import urlopen  # noqa
@@ -47,7 +51,7 @@ def urldefrag(url):
     else:
         defrag = url
         frag = ''
-    return defrag, frag
+    return DefragResult(defrag, frag)
 
 
 # flake8: noqa

jsonschema/tests/test_benchmarks.py

@@ -50,6 +50,10 @@ def test_V3_meta_schema(self):
 
             348 runs in 2.00 sec
                 ms/run: mean(5.73), std(0.38), MIN(5.50), MAX(8.00)
+        splitted_fragments: Time @ Xeon 3.2GHz (x 1.05 faster)::
+
+            290 runs in 2.00 sec
+                ms/run: mean(6.88), std(0.66), MIN(6.00), MAX(12.00)
         """
 
         stats = []
@@ -75,6 +79,10 @@ def test_V4_meta_schema(self):
 
             201 runs in 2.01 sec
                 ms/run: mean(9.95), std(0.57), MIN(9.50), MAX(12.50)
+        splitted_fragments: Time @ Xeon 3.2GHz (x 1.16 faster)::
+
+            191 runs in 2.01 sec
+                ms/run: mean(10.47), std(0.81), MIN(9.50), MAX(14.00)
         """
 
         stats = []
@@ -100,6 +108,10 @@ def test_both_meta_schemas(self):
 
             125 runs in 2.02 sec
                 ms/run: mean(16.00), std(0.88), MIN(15.00), MAX(19.00)
+        splitted_fragments: Time @ Xeon 3.2GHz (x 1.11 faster)::
+
+            115 runs in 2.00 sec
+                ms/run: mean(17.24), std(1.24), MIN(16.00), MAX(24.51)
         """
 
         v_classes = [Draft3Validator, Draft4Validator]
@@ -127,6 +139,10 @@ def test_ref_model(self):
 
             30 runs in 2.02 sec
                 ms/run: mean(65.05), std(2.11), MIN(62.01), MAX(71.51)
+        splitted_fragments: Time @ Xeon 3.2GHz (x 1.12 faster)::
+
+            19 runs in 2.10 sec
+                ms/run: mean(105.07), std(2.45), MIN(102.02), MAX(109.02)
         """
 
         stats = []

jsonschema/validators.py

@@ -11,7 +11,8 @@
 
 from jsonschema import _utils, _validators
 from jsonschema.compat import (
-    Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen,
+    Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, DefragResult,
+
     str_types, int_types, iteritems,
 )
 from jsonschema.exceptions import ErrorTree  # Backwards compatibility  # noqa
@@ -230,7 +231,7 @@ class RefResolver(object):
 
     :argument str base_uri: URI of the referring document
     :argument referrer: the actual referring document
-    :argument dict store: a mapping from URIs to documents to cache
+    :argument dict store: a mapping from URIs (without fragments!) to documents to cache
     :argument bool cache_remote: whether remote refs should be cached after
         first resolution
     :argument dict handlers: a mapping from URI schemes to functions that
@@ -241,6 +242,7 @@ class RefResolver(object):
     def __init__(
         self, base_uri, referrer, store=(), cache_remote=True, handlers=(),
     ):
+        base_uri = urldefrag(base_uri)
         self.base_uri = base_uri
         self.resolution_scope = base_uri
         # This attribute is not used, it is for backwards compatibility
@@ -251,11 +253,11 @@ def __init__(
 
         self.old_scopes = []
         self.store = _utils.URIDict(
-            (id, validator.META_SCHEMA)
+            (id, validator.META_SCHEMA)         ## IDs assumed pure urls (no fragments).
             for id, validator in iteritems(meta_schemas)
         )
         self.store.update(store)
-        self.store[base_uri] = referrer
+        self.store[base_uri.url] = referrer
 
     @classmethod
     def from_schema(cls, schema, *args, **kwargs):
@@ -287,19 +289,20 @@ def resolving(self, ref):
 
         """
 
-        full_uri = urljoin(self.resolution_scope, ref)
-        uri, fragment = urldefrag(full_uri)
-        if not uri:
-            uri = self.base_uri
+        ref = urldefrag(ref)
 
-        if uri in self.store:
-            document = self.store[uri]
-        else:
+        url = urljoin(self.resolution_scope.url, ref.url, allow_fragments=False) \
+                if ref.url else self.resolution_scope.url
+
+        try:
+            document = self.store[url]
+        except KeyError:
             try:
-                document = self.resolve_remote(uri)
+                document = self.resolve_remote(url)
             except Exception as exc:
                 raise RefResolutionError(exc)
 
+        uri = DefragResult(url, ref.fragment)
         old_base_uri, self.base_uri = self.base_uri, uri
         try:
             self.push_scope(uri)