Merge branch 'release/3.6' into develop

Author: thatbudakguy

CHANGELOG.rst

@@ -3,10 +3,20 @@
 CHANGELOG
 =========
 
+3.6
+---
+
+* accessibility: update hover and focus styles
+* chore: switch out semantic-ui for fomantic-ui and update js dependencies
+
 3.5
 ---
 
-* HathiTrust Data API client code has been removed in favor of using rsync. Configurations for **HATHITRUST_OAUTH_KEY** and  **HATHITRUST_OAUTH_SECRET** are no longer needed in local settings.
+* bugfix: refactor add new works from HathiTrust admin functionality to use rsync instead of API to work around restrictions on Google digitized
+* bugfix: improve overly-aggressive keyword search stemming
+* bugfix: remove page data from search index when suppressing works
+* New manage command ``index_pages`` to reindex pages more efficiently using multiprocessing
+* Refactored to use parasolr instead of SolrClient
 
 3.4
 ---

DEPLOYNOTES.rst

@@ -3,6 +3,14 @@
 Deploy and Upgrade notes
 ========================
 
+3.6
+---
+
+* Updates to javascript build tools used to compile fomantic-UI now require that
+  the version of nodejs be at least v10. This is already specified via the
+  README, but take care that deployment environments respect it or build
+  errors will occur.
+
 3.5
 ---
 
@@ -15,7 +23,15 @@ Deploy and Upgrade notes
      a new Solr configset, using the same name you put in local settings.
   3. Run `python manage.py solr_schema` to update (and optionally create)
      your configured Solr core with your configured configset.
-  4. Index data into your new solr core: `python manage.py index`
+  4. Index data into your new solr core::
+
+    python manage.py index -i work
+    python manage.py index_pages
+
+* HathiTrust Data API client code has been removed in favor of using rsync.
+  Configurations for **HATHITRUST_OAUTH_KEY** and  **HATHITRUST_OAUTH_SECRET**
+  are no longer needed in local settings.
+
 
 3.2
 ---

README.rst

@@ -17,19 +17,19 @@ at `<https://princeton-cdh.github.io/ppa-django/>`_.
    :target: https://github.com/Princeton-CDH/ppa-django/actions/workflows/unit-tests.yml
    :alt: Unit test status
 
-.. image:: https://codecov.io/gh/Princeton-CDH/ppa-django/branch/master/graph/badge.svg
+.. image:: https://codecov.io/gh/Princeton-CDH/ppa-django/branch/main/graph/badge.svg
    :target: https://codecov.io/gh/Princeton-CDH/ppa-django
    :alt: Code coverage
 
 .. image:: https://www.codefactor.io/repository/github/princeton-cdh/ppa-django/badge
    :target: https://www.codefactor.io/repository/github/princeton-cdh/ppa-django
    :alt: CodeFactor
 
-.. image:: https://requires.io/github/Princeton-CDH/ppa-django/requirements.svg?branch=master
-   :target: https://requires.io/github/Princeton-CDH/ppa-django/requirements/?branch=master
+.. image:: https://requires.io/github/Princeton-CDH/ppa-django/requirements.svg?branch=main
+   :target: https://requires.io/github/Princeton-CDH/ppa-django/requirements/?branch=main
    :alt: Requirements Status
 
-This repo uses `git-flow <https://github.com/nvie/gitflow>`_ conventions; **master**
+This repo uses `git-flow <https://github.com/nvie/gitflow>`_ conventions; **main**
 contains the most recent release, and work in progress will be on the **develop** branch.
 Pull requests should be made against develop.
 
@@ -199,7 +199,7 @@ Github Pages. After the build completes, push to GitHub from the ``docs`` folder
 
 License
 -------
-This project is licensed under the `Apache 2.0 License <https://github.com/Princeton-CDH/ppa-django/blob/master/LICENSE>`_.
+This project is licensed under the `Apache 2.0 License <https://github.com/Princeton-CDH/ppa-django/blob/main/LICENSE>`_.
 
 ©2019 Trustees of Princeton University.  Permission granted via
 Princeton Docket #20-3624 for distribution online under a standard Open Source

ppa/__init__.py

@@ -1,4 +1,4 @@
-__version_info__ = (3, 5, 0, 'dev')
+__version_info__ = (3, 6, 0, None)
 
 
 # Dot-connect all but the last. Last is dash-connected if not None.

ppa/archive/management/commands/generate_corpus.py

@@ -154,10 +154,6 @@ def __init__(self, name, doc_limit=-1, preprocess_fns=None, pbar=True):
         results = SolrQuerySet() \
             .facet(SolrCorpus.DOC_ID_FIELD, limit=self.doc_limit)
 
-        print('*** solrqueryset')
-        print(results)
-        print(results.get_facets().facet_fields)
-
         """
         An OrderedDict of doc_id => page count mapping
         An OrderedDict is important here in case we want to save document-level
@@ -184,9 +180,6 @@ def __iter__(self):
                                format(doc_id))
                 continue
 
-            print('*** solrqueryset')
-            print(SolrQuerySet)
-
             result = SolrQuerySet() \
                 .search(**{SolrCorpus.DOC_ID_FIELD: doc_id}) \
                 .order_by(SolrCorpus.PAGE_ORDER_FIELD)

ppa/archive/templates/archive/add_from_hathi.html

@@ -35,9 +35,9 @@ <h2>Processed {{ results|length }} HathiTrust Identifier{{ results|pluralize}}.<
             <td>{{ htid }}</td>
             <td>{{ result }}</td>
             {% if 'Success' in result or htid in existing_ids %}
-            <td><a href="{{ admin_urls|dict_item:htid }}" target="_blank">
+            <td><a href="{{ admin_urls|dict_item:htid }}" target="_blank" rel="noopener noreferrer">
                 View in Admin</a></td>
-            <td><a href="{% url 'archive:detail' htid %}" target="_blank">
+            <td><a href="{% url 'archive:detail' htid %}" target="_blank" rel="noopener noreferrer">
                 View on Site</a></td>
             {% else %}
             <td></td><td></td>

ppa/archive/templates/archive/digitizedwork_detail.html

@@ -87,7 +87,7 @@ <h1 class="header">{{ object.title }}</h1>
                 <th scope="row">View{% if object.source == object.OTHER %} external record
                     {% else %} on {{ object.get_source_display }}{% endif %}</th>
                 <td>
-                    <a href="{{ object.source_url }}" target="_blank">{{ object.source_id }}</a>
+                    <a href="{{ object.source_url }}" target="_blank" rel="noopener noreferrer">{{ object.source_id }}</a>
                 </td>
             </tr>
             {% endif %}

ppa/archive/templates/archive/snippets/page_preview.html

@@ -11,7 +11,7 @@
 <div class="page">
     <div class="preview">
         <a href="{% page_url item_id page.order %}"
-          target=_blank>
+          target="_blank" rel="noopener noreferrer">
           {% page_image_url item_id page.order 225 as 1x_img %}
           {% page_image_url item_id page.order 450 as 2x_img %}
           {# img with data-src/srcset attributes for lazy-loading  #}
@@ -27,7 +27,7 @@
     {% if page.title %}<a>p. {{ page.title }}</a>{% endif %}
     <div class="snippets">
           <p class="page-number">
-            <a href="{% page_url item_id page.order %}" target=_blank>
+            <a href="{% page_url item_id page.order %}" target="_blank" rel="noopener noreferrer">
               p. {{ page.label }}
             </a>
           </p>

requirements.lock

@@ -1,136 +1,59 @@
-alabaster==0.7.12
-appnope==0.1.0
-atomicwrites==1.3.0
-attrs==19.1.0
-Babel==2.7.0
-backcall==0.1.0
-beautifulsoup4==4.6.0
-bleach==3.1.0
-boto==2.49.0
-boto3==1.9.174
-botocore==1.12.174
-cached-property==1.5.1
-certifi==2019.3.9
-chardet==3.0.4
-coverage==4.5.3
-cycler==0.10.0
-Cython==0.29.13
-decorator==4.4.0
-defusedxml==0.6.0
-dictionaries==0.0.1
-Django==2.2.12
-django-apptemplates==1.4
-django-cas-ng==3.6.0
-django-csp==3.5
-django-debug-toolbar==2.0
-django-fullurl==1.0
-django-grappelli==2.13.3
-django-modelcluster==5.0.1
+attrdict==2.0.1
+beautifulsoup4==4.8.2
+bleach==3.3.0
+cached-property==1.5.2
+certifi==2020.12.5
+chardet==4.0.0
+dataclasses==0.8
+dictionaries==0.0.2
+Django==2.2.23
+django-apptemplates==1.5
+django-cas-ng==4.1.1
+django-csp==3.7
+django-fullurl==1.1
+django-grappelli==2.15.1
+django-modelcluster==5.1
 django-semanticui-forms==1.6.5
-django-taggit==1.2.0
-django-treebeard==4.3
-django-webpack-loader==0.6.0
-djangorestframework==3.9.4
-docutils==0.14
-draftjs-exporter==2.1.6
-entrypoints==0.3
+django-taggit==1.4.0
+django-treebeard==4.5.1
+django-webpack-loader==1.0.0
+djangorestframework==3.12.4
+draftjs-exporter==2.1.7
 eulxml==1.1.3
-funcy==1.12
-future==0.17.1
-gensim==3.7.3
-html5lib==1.0.1
-idna==2.8
-imagesize==1.1.0
-importlib-metadata==0.17
-ipykernel==5.1.2
-ipython==7.7.0
-ipython-genutils==0.2.0
-ipywidgets==7.5.1
-jedi==0.15.1
-Jinja2==2.10.1
-jmespath==0.9.4
-joblib==0.13.2
-jsonschema==3.0.2
-jupyter==1.0.0
-jupyter-client==5.3.1
-jupyter-console==6.0.0
-jupyter-core==4.5.0
-kazoo==2.2.1
-kiwisolver==1.1.0
-l18n==2018.5
-ldap3==2.6
-lxml==4.3.3
-MarkupSafe==1.1.1
-matplotlib==3.1.1
-mistune==0.8.4
-more-itertools==7.0.0
-mysqlclient==1.4.2.post1
-nbconvert==5.6.0
-nbformat==4.4.0
-notebook==6.0.0
-numexpr==2.6.9
-numpy==1.16.4
-oauthlib==3.0.1
-packaging==19.0
+gensim==4.0.1
+html5lib==1.1
+idna==2.10
+l18n==2020.6.1
+ldap3==2.9
+lxml==4.6.3
+mysqlclient==2.0.3
+numpy==1.19.5
+oauthlib==3.1.0
+packaging==20.9
 Pairtree==0.8.1
-pandas==0.24.2
-pandocfilters==1.4.2
-parso==0.5.1
-pexpect==4.7.0
-pickleshare==0.7.5
-Pillow==5.4.1
-pluggy==0.12.0
+parasolr==0.6.1
+Pillow==7.2.0
 ply==3.11
-progressbar2==3.42.0
-prometheus-client==0.7.1
-prompt-toolkit==2.0.9
-ptyprocess==0.6.0
-pucas==0.6.0
-py==1.8.0
-py-flags==1.1.2
-pyasn1==0.4.5
-Pygments==2.4.2
-pyLDAvis==2.1.2
-pymarc==3.1.13
-pyparsing==2.4.0
-pyrsistent==0.15.4
-pytest==5.4.1
-pytest-cov==2.7.1
-pytest-django==3.5.1
-python-cas==1.4.0
-python-dateutil==2.8.0
-python-utils==2.3.0
-pytz==2019.1
-pyzmq==18.1.0
-qtconsole==4.5.3
-requests==2.22.0
-requests-oauthlib==1.2.0
-s3transfer==0.2.1
-scipy==1.3.0
-Send2Trash==1.5.0
-six==1.12.0
-smart-open==1.8.4
-snowballstemmer==1.2.1
--e [email protected]:rlskoeser/SolrClient.git@7420ab3ca97be6fcc178d74e10c136b73b332edd#egg=SolrClient
-Sphinx==2.1.2
-sphinxcontrib-applehelp==1.0.1
-sphinxcontrib-devhelp==1.0.1
-sphinxcontrib-htmlhelp==1.0.2
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.2
-sphinxcontrib-serializinghtml==1.1.3
-sqlparse==0.3.0
-termcolor==1.1.0
-terminado==0.8.2
-testpath==0.4.2
-tornado==6.0.3
-traitlets==4.3.2
-Unidecode==1.0.23
-urllib3==1.25.3
-wagtail==2.8.1
-Wand==0.5.4
-wcwidth==0.1.7
+progressbar2==3.53.1
+pucas==0.7.0
+py-flags==1.1.4
+pyasn1==0.4.8
+pymarc==4.1.0
+pyparsing==2.4.7
+python-cas==1.5.0
+python-utils==2.5.6
+pytz==2021.1
+requests==2.25.1
+requests-oauthlib==1.3.0
+scipy==1.5.4
+six==1.16.0
+smart-open==5.0.0
+soupsieve==2.2.1
+sqlparse==0.4.1
+Unidecode==1.2.0
+urllib3==1.26.4
+wagtail==2.9.3
+Wand==0.6.6
 webencodings==0.5.1
-widgetsnbextension==3.5.1
 Willow==1.3
-zipp==0.5.1
+XlsxWriter==1.4.3