Add SerpMetadata fields to Metadata (#123)

docs/reference/components.rst

@@ -95,7 +95,7 @@ Item metadata components
 
 .. autoclass:: zyte_common_items.Metadata(**kwargs)
    :show-inheritance:
-   :members: dateDownloaded, probability, searchText, validationMessages
+   :members: dateDownloaded, displayedQuery, probability, searchedQuery, searchText, totalOrganicResults, validationMessages, get_date_downloaded_parsed
 
 .. autoclass:: zyte_common_items.ProbabilityMetadata(**kwargs)
    :show-inheritance:
@@ -109,6 +109,10 @@ Item metadata components
    :show-inheritance:
    :members: probability, dateDownloaded, validationMessages, get_date_downloaded_parsed
 
+.. autoclass:: zyte_common_items.SearchMetadata(**kwargs)
+   :show-inheritance:
+   :members: dateDownloaded, probability, searchText, validationMessages, get_date_downloaded_parsed
+
 .. autoclass:: zyte_common_items.BaseMetadata(**kwargs)
    :show-inheritance:
    :members: cast

tests/test_components.py

@@ -1,11 +1,13 @@
 import datetime
 
+import attrs
 from web_poet import RequestUrl
 
 from zyte_common_items import (
     Address,
     AggregateRating,
     Amenity,
+    BaseMetadata,
     BaseSalary,
     Breadcrumb,
     BusinessPlaceMetadata,
@@ -72,6 +74,31 @@ def test_metadata_get_date_downloaded():
     )
 
 
+def get_all_subclasses(cls):
+    subclasses = set()
+    for subclass in cls.__subclasses__():
+        subclasses.add(subclass)
+        subclasses.update(get_all_subclasses(subclass))
+    return subclasses
+
+
+def test_metadata_fields():
+    """Metadata must contain a superset of the fields of all metadata
+    classes."""
+    superset = set(attrs.fields_dict(Metadata))
+    for cls in get_all_subclasses(BaseMetadata):
+        subset = set(attrs.fields_dict(cls))
+        assert subset.issubset(
+            superset
+        ), f"Metadata is missing some fields from {cls.__name__}: {subset - superset}"
+
+
+def test_metadata_subclasses():
+    """Metadata should not be subclassed, since its fields will grow as new
+    specific metadata classes are added."""
+    assert not get_all_subclasses(Metadata)
+
+
 def test_named_link_optional_fields():
     NamedLink(name="foo")
     NamedLink(url="https://example.com")

zyte_common_items/__init__.py

@@ -29,6 +29,7 @@
     Reactions,
     RealEstateArea,
     Request,
+    SearchMetadata,
     SocialMediaPostAuthor,
     StarRating,
     Topic,

zyte_common_items/components/__init__.py

@@ -18,6 +18,7 @@
     Metadata,
     MetadataT,
     ProbabilityMetadata,
+    SearchMetadata,
 )
 from .ratings import AggregateRating, StarRating
 from .real_estate import RealEstateArea

zyte_common_items/components/metadata.py

@@ -74,13 +74,28 @@ class DetailsMetadata(ListMetadata):
 
 
 @attrs.define(kw_only=True)
-class Metadata(DetailsMetadata):
+class SearchMetadata(DetailsMetadata):
+    """Minimal metadata for classes of items that can declare search
+    metadata."""
+
+    #: The search text used to find the item.
+    searchText: Optional[str] = None
+
+
+@attrs.define(kw_only=True)
+class Metadata(SearchMetadata):
     """Generic metadata class.
 
     It defines all attributes of metadata classes for specific item types, so
     that it can be used during extraction instead of a more specific class, and
     later converted to the corresponding, more specific metadata class.
     """
 
-    #: The search text used to find the item.
-    searchText: Optional[str] = None
+    #: Search query as seen in the webpage.
+    displayedQuery: Optional[str] = None
+
+    #: Search query as specified in the input URL.
+    searchedQuery: Optional[str] = None
+
+    #: Total number of organic results reported by the search engine.
+    totalOrganicResults: Optional[int] = None

zyte_common_items/items/business_place.py

@@ -9,17 +9,17 @@
     AggregateRating,
     Amenity,
     Image,
-    Metadata,
     NamedLink,
     OpeningHoursItem,
     ParentPlace,
+    SearchMetadata,
     StarRating,
 )
 from zyte_common_items.converters import to_metadata_optional, url_to_str_optional
 
 
 @attrs.define(kw_only=True)
-class BusinessPlaceMetadata(Metadata):
+class BusinessPlaceMetadata(SearchMetadata):
     """Metadata class for :data:`zyte_common_items.BusinessPlace.metadata`."""
 
 

zyte_common_items/items/job_posting.py

@@ -7,13 +7,13 @@
     BaseSalary,
     HiringOrganization,
     JobLocation,
-    Metadata,
+    SearchMetadata,
 )
 from zyte_common_items.converters import to_metadata_optional, url_to_str
 
 
 @attrs.define(kw_only=True)
-class JobPostingMetadata(Metadata):
+class JobPostingMetadata(SearchMetadata):
     """Metadata class for :data:`zyte_common_items.JobPosting.metadata`."""
 
 

zyte_common_items/items/social_media_post.py

@@ -3,12 +3,17 @@
 import attrs
 
 from zyte_common_items.base import Item
-from zyte_common_items.components import Metadata, Reactions, SocialMediaPostAuthor, Url
+from zyte_common_items.components import (
+    Reactions,
+    SearchMetadata,
+    SocialMediaPostAuthor,
+    Url,
+)
 from zyte_common_items.converters import to_metadata_optional, url_to_str
 
 
 @attrs.define(kw_only=True)
-class SocialMediaPostMetadata(Metadata):
+class SocialMediaPostMetadata(SearchMetadata):
     """Metadata class for :data:`zyte_common_items.SocialMediaPost.metadata`."""