Update script and example for unescaping HTML.

Author: gkellogg

common/extract-examples.rb

@@ -15,6 +15,7 @@
 require 'fileutils'
 require 'colorize'
 require 'yaml'
+require 'cgi'
 
 PREFIXES = {
   dc:     "http://purl.org/dc/terms/",
@@ -49,8 +50,8 @@
 # Remove highlighting and commented out sections
 def justify(str)
   str = str.
-    sub(/^\s*<!--\s*$/, '').
-    sub(/^\s*-->\s*$/, '').
+    gsub(/^\s*<!--\s*$/, '').
+    gsub(/^\s*-->\s*$/, '').
     gsub('****', '').
     gsub(/####([^#]*)####/, '')
 
@@ -222,7 +223,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
         examples[title] = {
           title: title,
           filename: fn,
-          content: content,
+          content: content.to_s.gsub(/^\s*< !\s*-\s*-/, '<!--').gsub(/-\s*- >/, '-->'),
           content_type: element.attr('data-content-type'),
           number: example_number,
           ext: ext,
@@ -302,6 +303,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
     # Perform example syntactic validation based on extension
     case ex[:ext]
     when 'json', 'jsonld', 'jsonldf'
+      content = CGI.unescapeHTML(content)
       begin
         ::JSON.parse(content)
       rescue JSON::ParserError => exception
@@ -325,22 +327,17 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
         ex[:base] = html_base.to_s if html_base
 
         script_content = doc.at_xpath(xpath)
-        if script_content
-          # Remove (faked) XML comments and unescape sequences
-          content = script_content
-            .inner_html
-            .sub(/^\s*< !\s*-\s*-/, '')
-            .sub(/-\s*- >\s*$/, '')
-            .gsub(/&lt;/, '<')
-        end
-          
+        
+        # Remove (faked) XML comments and unescape sequences
+        content = CGI.unescapeHTML(script_content.inner_html) if script_content          
       rescue Nokogiri::XML::SyntaxError => exception
         errors << "Example #{ex[:number]} at line #{ex[:line]} parse error: #{exception.message}"
         $stdout.write "F".colorize(:red)
         next
       end
     when 'table'
-      # already in parsed form
+      doc = Nokogiri::HTML.parse(content) {|c| c.strict}
+      content = CGI.unescapeHTML(doc.inner_html)
     when 'ttl', 'trig'
       begin
         reader_errors = []
@@ -443,10 +440,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
       # Set argument to referenced content to be parsed
       args[0] = if examples[ex[:result_for]][:ext] == 'html' && method == :expand
         # If we are expanding, and the reference is HTML, find the first script element.
-        doc = Nokogiri::HTML.parse(
-          examples[ex[:result_for]][:content]
-          .sub(/^\s*< !\s*-\s*-/, '')
-          .sub(/-\s*- >\s*$/, ''))
+        doc = Nokogiri::HTML.parse(examples[ex[:result_for]][:content])
 
         # Get base from document, if present
         html_base = doc.at_xpath('/html/head/base/@href')
@@ -458,15 +452,10 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
           $stdout.write "F".colorize(:red)
           next
         end
-        StringIO.new(script_content
-          .inner_html
-          .gsub(/&lt;/, '<'))
+        StringIO.new(CGI.unescapeHTML(script_content.inner_html))
       elsif examples[ex[:result_for]][:ext] == 'html' && ex[:target]
         # Only use the targeted script
-        doc = Nokogiri::HTML.parse(
-          examples[ex[:result_for]][:content]
-          .sub(/^\s*< !\s*-\s*-/, '')
-          .sub(/-\s*- >\s*$/, ''))
+        doc = Nokogiri::HTML.parse(examples[ex[:result_for]][:content])
         script_content = doc.at_xpath(xpath)
         unless script_content
           errors << "Example #{ex[:number]} at line #{ex[:line]} references example #{ex[:result_for].inspect} with no JSON-LD script element"
@@ -565,7 +554,8 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
           $stderr.puts "expected:\n" + expected.to_trig if verbose
         when 'table'
           expected = begin
-            table_to_dataset(content)
+            doc = Nokogiri::HTML.parse(content)
+            table_to_dataset(doc)
           rescue
             errors << "Example #{ex[:number]} at line #{ex[:line]} raised error reading table: #{$!}"
             RDF::Dataset.new

index.html

@@ -8702,7 +8702,6 @@ <h3>Graph Containers</h3>
       <div class="selectors">
         <button class="selected" data-selects="original">Original</button>
         <button data-selects="expanded">Expanded</button>
-        <button data-selects="statements">Statements</button>
         <button data-selects="turtle">Turtle</button>
       </div>
       <pre class="original selected" data-transform="updateExample"
@@ -8720,36 +8719,29 @@ <h3>Graph Containers</h3>
       </pre>
       <pre class="expanded"
            data-transform="updateExample"
-           data-result-for="Embedding JSON-LD in HTML with comments-original">
+           data-result-for="Embedding JSON-LD containing HTML in HTML-original">
       <!--
       [{
-        "@type": "http://schema.org/WebPageElement",
+        "@type": ["http://schema.org/WebPageElement"],
         "http://schema.org/name": [{"@value": "Encoding Issues"}],
         "http://schema.org/description": [
-          {"@value": "Issues list such as unescaped &lt;/script&gt; or --&gt;"}
+          {"@value": "Issues list such as unescaped </script> or -- >"}
         ]
       }]
       -->
       </pre>
-      <table class="statements"
-             data-result-for="Embedding JSON-LD in HTML with comments-expanded"
-             data-to-rdf>
-        <thead><tr><th>Subject</th><th>Property</th><th>Value</th><th>Value Type</th></tr></thead>
-        <tbody>
-          <tr><td>_:b0</td><td>schema:name</td><td>Encoding Issues</td><td>&nbsp;</td></tr>
-          <tr><td>_:b0</td><td>schema:description</td><td>Issues list such as unescaped &amp;lt;/script&amp;gt; or --&amp;gt;</td><td>&nbsp;</td></tr>
-        </tbody>
-      </table>
       <pre class="turtle"
            data-content-type="text/turtle"
            data-transform="updateExample"
-           data-result-for="Embedding JSON-LD in HTML with comments-expanded"
+           data-result-for="Embedding JSON-LD containing HTML in HTML-expanded"
            data-to-rdf>
       <!--
       @prefix schema: <http://schema.org/> .
 
-      <> schema:name "Encoding Issues";
-         schema:description "Issues list such as unescaped &lt;/script&gt; or --&gt;" .
+      [ a schema:WebPageElement;
+        schema:name "Encoding Issues";
+        schema:description "Issues list such as unescaped </script> or -- >"
+      ] .
       -->
       </pre>
     </aside>