Add subscriptions graphql schema (#275)

* Add subscriptions graphql schema

- use processes_resolver in subscription schema to resolve processes of a subscription.
- use subscriptions_resolver in processes schema to resolve subscriptions of a process.
- add subscriptions db filters and sorting.
- add generic filters to remove duplicated code.

* Add depends_on and in_use_by queries to the subscription query

* Improve date filtering to be less trict with full date

* fix pipeline issues

* Add graphql tests for subscriptions and update processes tests

* Improve db generic range filter

* Rename filter functions by column for each filter resource

* Remove customerId in graphql resources and filtering.

also remove it from tests.
add filter_by to new variable instead of overwriting it.

* add subscriptions resource with resolver to products resource

fix generic_values_in_column_filter by lowercasing the column value.

* Remove ye and insync from the generic bool filter

* Remove customer id filter test from processes filterable endpoint

* Improve range filter with use of partial functool

* Remove unnecessary _range from resolvers

* Rename resolve_subscription to resolve_subscriptions

- improve descriptions of relation fields.
- update single subscription tests to check product blocks.

Author: tjeerddie

orchestrator/.stignore

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2023 SURF.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -53,7 +53,7 @@ def _apply_filters(
         query: QueryType, filter_by: Iterator[Filter], handle_filter_error: CallableErrorHander
     ) -> QueryType:
         for item in filter_by:
-            field = item.field.lower()
+            field = item.field
             filter_fn = valid_filter_functions_by_column[field]
             try:
                 query = filter_fn(query, item.value)
@@ -63,6 +63,12 @@ def _apply_filters(
                     field=field,
                     value=item.value,
                 )
+            except ValueError as exception:
+                handle_filter_error(
+                    str(exception),
+                    field=field,
+                    value=item.value,
+                )
         return query
 
     return _apply_filters

orchestrator/db/filters/filters.py

@@ -0,0 +1,23 @@
+from orchestrator.db.filters.generic_filters.bool_filter import generic_bool_filter
+from orchestrator.db.filters.generic_filters.is_like_filter import generic_is_like_filter
+from orchestrator.db.filters.generic_filters.range_filter import (
+    RANGE_TYPES,
+    convert_to_date,
+    convert_to_int,
+    generic_range_filter,
+    generic_range_filters,
+    get_filter_value_convert_function,
+)
+from orchestrator.db.filters.generic_filters.values_in_column_filter import generic_values_in_column_filter
+
+__all__ = [
+    "RANGE_TYPES",
+    "convert_to_date",
+    "convert_to_int",
+    "get_filter_value_convert_function",
+    "generic_range_filter",
+    "generic_range_filters",
+    "generic_is_like_filter",
+    "generic_values_in_column_filter",
+    "generic_bool_filter",
+]

orchestrator/db/filters/generic_filters/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2019-2023 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable
+
+from sqlalchemy import Column
+
+from orchestrator.db.database import SearchQuery
+
+
+def generic_bool_filter(field: Column) -> Callable[[SearchQuery, str], SearchQuery]:
+    def bool_filter(query: SearchQuery, value: str) -> SearchQuery:
+        value_as_bool = value.lower() in ("yes", "y", "true", "1", "ja", "j")
+        return query.filter(field.is_(value_as_bool))
+
+    return bool_filter

orchestrator/db/filters/generic_filters/bool_filter.py

@@ -0,0 +1,26 @@
+# Copyright 2019-2023 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Callable
+
+from sqlalchemy import Column, String, cast
+
+from orchestrator.db.database import SearchQuery
+
+
+def generic_is_like_filter(field: Column) -> Callable[[SearchQuery, str], SearchQuery]:
+    def like_filter(query: SearchQuery, value: str) -> SearchQuery:
+        return query.filter(cast(field, String).ilike("%" + value + "%"))
+
+    return like_filter

orchestrator/db/filters/generic_filters/is_like_filter.py

@@ -0,0 +1,80 @@
+# Copyright 2019-2023 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from datetime import datetime
+from functools import partial
+from typing import Callable, Optional
+
+import pytz
+from dateutil.parser import parse
+from sqlalchemy import Column
+
+from orchestrator.db.database import SearchQuery
+
+# from sqlalchemy.sql.expression import ColumnOperators
+from orchestrator.utils.helpers import to_camel
+
+RANGE_TYPES = {
+    "gt": Column.__gt__,
+    "gte": Column.__ge__,
+    "lt": Column.__lt__,
+    "lte": Column.__le__,
+    "ne": Column.__ne__,
+}
+
+
+def convert_to_date(value: str) -> datetime:
+    """Parse iso 8601 date from string to datetime.
+
+    Example date: "2022-07-21T03:40:48+00:00"
+    """
+    try:
+        return parse(value).replace(tzinfo=pytz.UTC)
+    except ValueError:
+        raise ValueError(f"{value} is not a valid date")
+
+
+def convert_to_int(value: str) -> int:
+    try:
+        return int(value)
+    except ValueError:
+        raise ValueError(f"{value} is not a valid integer")
+
+
+def get_filter_value_convert_function(field: Column) -> Callable:
+    if field.type.python_type == datetime:
+        return convert_to_date
+    if field.type.python_type == int:
+        return convert_to_int
+    return lambda x: x
+
+
+def generic_range_filter(range_type_fn: Callable, field: Column) -> Callable[[SearchQuery, str], SearchQuery]:
+    filter_operator = partial(range_type_fn, field)
+    convert_filter_value = get_filter_value_convert_function(field)
+
+    def use_filter(query: SearchQuery, value: str) -> SearchQuery:
+        converted_value = convert_filter_value(value)
+        return query.filter(filter_operator(converted_value))
+
+    return use_filter
+
+
+def generic_range_filters(
+    column: Column, column_alias: Optional[str] = None
+) -> dict[str, Callable[[SearchQuery, str], SearchQuery]]:
+    column_name = to_camel(column_alias or column.name)
+
+    return {
+        f"{column_name}{operator.capitalize()}": generic_range_filter(operator_fn, column)
+        for operator, operator_fn in RANGE_TYPES.items()
+    }

orchestrator/db/filters/generic_filters/range_filter.py

@@ -0,0 +1,27 @@
+# Copyright 2019-2023 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Callable
+
+from sqlalchemy import Column, func
+
+from orchestrator.db.database import SearchQuery
+
+
+def generic_values_in_column_filter(field: Column) -> Callable[[SearchQuery, str], SearchQuery]:
+    def list_filter(query: SearchQuery, value: str) -> SearchQuery:
+        values = [s.lower() for s in value.split("-")]
+        return query.filter(func.lower(field).in_(values))
+
+    return list_filter

orchestrator/db/filters/generic_filters/values_in_column_filter.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF.
+# Copyright 2019-2023 SURF.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -11,65 +11,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from http import HTTPStatus
 from typing import Callable
-from uuid import UUID
 
 import structlog
-from sqlalchemy import String, cast
 
-from orchestrator.api.error_handling import raise_status
 from orchestrator.db import ProcessSubscriptionTable, ProcessTable, ProductTable, SubscriptionTable, db
 from orchestrator.db.database import SearchQuery
 from orchestrator.db.filters.filters import generic_filter
+from orchestrator.db.filters.generic_filters import (
+    generic_bool_filter,
+    generic_is_like_filter,
+    generic_values_in_column_filter,
+)
 
 logger = structlog.get_logger(__name__)
 
 
-def pid_filter(query: SearchQuery, value: str) -> SearchQuery:
-    return query.filter(cast(ProcessTable.pid, String).ilike("%" + value + "%"))
-
-
-def is_task_filter(query: SearchQuery, value: str) -> SearchQuery:
-    value_as_bool = value.lower() in ("yes", "y", "ye", "true", "1", "ja")
-    return query.filter(ProcessTable.is_task.is_(value_as_bool))
-
-
-def assignee_filter(query: SearchQuery, value: str) -> SearchQuery:
-    assignees = value.split("-")
-    return query.filter(ProcessTable.assignee.in_(assignees))
-
-
-def status_filter(query: SearchQuery, value: str) -> SearchQuery:
-    statuses = value.split("-")
-    return query.filter(ProcessTable.last_status.in_(statuses))
-
-
-def workflow_filter(query: SearchQuery, value: str) -> SearchQuery:
-    return query.filter(ProcessTable.workflow.ilike("%" + value + "%"))
-
-
-def creator_filter(query: SearchQuery, value: str) -> SearchQuery:
-    return query.filter(ProcessTable.created_by.ilike("%" + value + "%"))
-
-
-def organisation_filter(query: SearchQuery, value: str) -> SearchQuery:
-    try:
-        value_as_uuid = UUID(value)
-    except (ValueError, AttributeError):
-        msg = f"Not a valid organisation, must be a UUID: '{value}'"
-        logger.debug(msg)
-        raise_status(HTTPStatus.BAD_REQUEST, msg)
-
-    process_subscriptions = (
-        db.session.query(ProcessSubscriptionTable)
-        .join(SubscriptionTable)
-        .filter(SubscriptionTable.customer_id == value_as_uuid)
-        .subquery()
-    )
-    return query.filter(ProcessTable.pid == process_subscriptions.c.pid)
-
-
 def product_filter(query: SearchQuery, value: str) -> SearchQuery:
     process_subscriptions = (
         db.session.query(ProcessSubscriptionTable)
@@ -101,6 +58,13 @@ def subscriptions_filter(query: SearchQuery, value: str) -> SearchQuery:
     return query.filter(ProcessTable.pid == process_subscriptions.c.pid)
 
 
+def subscription_id_filter(query: SearchQuery, value: str) -> SearchQuery:
+    process_subscriptions = db.session.query(ProcessSubscriptionTable).join(SubscriptionTable)
+    process_subscriptions = generic_is_like_filter(SubscriptionTable.subscription_id)(process_subscriptions, value)
+    process_subscriptions = process_subscriptions.subquery()
+    return query.filter(ProcessTable.pid == process_subscriptions.c.pid)
+
+
 def target_filter(query: SearchQuery, value: str) -> SearchQuery:
     targets = value.split("-")
     process_subscriptions = (
@@ -111,19 +75,19 @@ def target_filter(query: SearchQuery, value: str) -> SearchQuery:
     return query.filter(ProcessTable.pid == process_subscriptions.c.pid)
 
 
-VALID_FILTER_FUNCTIONS_BY_COLUMN: dict[str, Callable[[SearchQuery, str], SearchQuery]] = {
-    "pid": pid_filter,
-    "istask": is_task_filter,
-    "assignee": assignee_filter,
-    "status": status_filter,
-    "workflow": workflow_filter,
-    "creator": creator_filter,
-    "organisation": organisation_filter,
+PROCESS_FILTER_FUNCTIONS_BY_COLUMN: dict[str, Callable[[SearchQuery, str], SearchQuery]] = {
+    "pid": generic_is_like_filter(ProcessTable.pid),
+    "istask": generic_bool_filter(ProcessTable.is_task),
+    "assignee": generic_values_in_column_filter(ProcessTable.assignee),
+    "status": generic_values_in_column_filter(ProcessTable.last_status),
+    "workflow": generic_is_like_filter(ProcessTable.workflow),
+    "creator": generic_is_like_filter(ProcessTable.created_by),
     "product": product_filter,
     "tag": tag_filter,
     "subscription": subscriptions_filter,
+    "subscriptionId": subscription_id_filter,
     "target": target_filter,
 }
 
 
-filter_processes = generic_filter(VALID_FILTER_FUNCTIONS_BY_COLUMN)
+filter_processes = generic_filter(PROCESS_FILTER_FUNCTIONS_BY_COLUMN)

orchestrator/db/filters/process.py

@@ -1,54 +1,29 @@
 from typing import Callable
 
 import structlog
-from sqlalchemy import String, cast
 
 from orchestrator.db import ProductBlockTable, ProductTable
-from orchestrator.db.database import BaseModelMeta, SearchQuery
+from orchestrator.db.database import SearchQuery
 from orchestrator.db.filters import generic_filter
+from orchestrator.db.filters.generic_filters import generic_is_like_filter, generic_values_in_column_filter
 
 logger = structlog.get_logger(__name__)
 
 
-def field_filter(table: BaseModelMeta, field: str) -> Callable:
-    def _field_filter(query: SearchQuery, value: str) -> SearchQuery:
-        logger.debug("Called _field_filter(...)", query=query, table=table, field=field, value=value)
-        return query.filter(getattr(table, field).ilike("%" + value + "%"))
-
-    return _field_filter
-
-
-def list_filter(table: BaseModelMeta, field: str) -> Callable:
-    def _list_filter(query: SearchQuery, value: str) -> SearchQuery:
-        values = value.split("-")
-        return query.filter(getattr(table, field).in_(values))
-
-    return _list_filter
-
-
-def id_filter(query: SearchQuery, value: str) -> SearchQuery:
-    return query.filter(cast(ProductTable.product_id, String).ilike("%" + value + "%"))
-
-
 def product_block_filter(query: SearchQuery, value: str) -> SearchQuery:
     """Filter ProductBlocks by '-'-separated list of Product block 'name' (column) values."""
     blocks = value.split("-")
     return query.filter(ProductTable.product_blocks.any(ProductBlockTable.name.in_(blocks)))
 
 
-# TODO
-# def date_filter(start: str, end: Optional[str]) -> None:
-#     pass
-
-
-VALID_FILTER_FUNCTIONS_BY_COLUMN: dict[str, Callable[[SearchQuery, str], SearchQuery]] = {
-    "product_id": id_filter,
-    "name": field_filter(ProductTable, "name"),
-    "description": field_filter(ProductTable, "description"),
-    "product_type": field_filter(ProductTable, "product_type"),
-    "status": list_filter(ProductTable, "status"),
-    "tag": list_filter(ProductTable, "tag"),
+PRODUCT_FILTER_FUNCTIONS_BY_COLUMN: dict[str, Callable[[SearchQuery, str], SearchQuery]] = {
+    "product_id": generic_is_like_filter(ProductTable.product_id),
+    "name": generic_is_like_filter(ProductTable.name),
+    "description": generic_is_like_filter(ProductTable.description),
+    "product_type": generic_is_like_filter(ProductTable.product_type),
+    "status": generic_values_in_column_filter(ProductTable.status),
+    "tag": generic_values_in_column_filter(ProductTable.tag),
     "product_blocks": product_block_filter,
 }
 
-filter_products = generic_filter(VALID_FILTER_FUNCTIONS_BY_COLUMN)
+filter_products = generic_filter(PRODUCT_FILTER_FUNCTIONS_BY_COLUMN)