Skip to content

Commit

Permalink
Merge pull request #108 from zytedata/job-navigation
Browse files Browse the repository at this point in the history
Add job posting navigation classes.
  • Loading branch information
wRAR authored Oct 2, 2024
2 parents 047465c + a217dd5 commit f3d6356
Show file tree
Hide file tree
Showing 8 changed files with 132 additions and 3 deletions.
12 changes: 9 additions & 3 deletions docs/reference/items.rst
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ Article
.. autoclass:: zyte_common_items.ArticleMetadata(**kwargs)
:members: dateDownloaded, probability, validationMessages


Article list
============

Expand Down Expand Up @@ -87,7 +86,6 @@ Business place
.. autoclass:: zyte_common_items.BusinessPlaceMetadata(**kwargs)
:members: dateDownloaded, probability, searchText, validationMessages


Real estate
===========

Expand All @@ -108,6 +106,15 @@ Job posting
.. autoclass:: zyte_common_items.JobPostingMetadata(**kwargs)
:members: dateDownloaded, probability, validationMessages

Job posting navigation
======================

.. autoclass:: zyte_common_items.JobPostingNavigation(**kwargs)
:members:
:inherited-members:

.. autoclass:: zyte_common_items.JobPostingNavigationMetadata(**kwargs)
:members: dateDownloaded, validationMessages

Search engine results
=====================
Expand All @@ -119,7 +126,6 @@ Search engine results
.. autoclass:: zyte_common_items.SerpMetadata(**kwargs)
:members: dateDownloaded, displayedQuery, searchedQuery, totalOrganicResults, validationMessages


Social media post
=================

Expand Down
12 changes: 12 additions & 0 deletions docs/reference/pages.rst
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,18 @@ Job posting
.. autoclass:: zyte_common_items.AutoJobPostingPage(**kwargs)
:show-inheritance:

Job posting navigation
======================

.. autoclass:: zyte_common_items.BaseJobPostingNavigationPage(**kwargs)
:show-inheritance:

.. autoclass:: zyte_common_items.JobPostingNavigationPage(**kwargs)
:show-inheritance:

.. autoclass:: zyte_common_items.AutoJobPostingNavigationPage(**kwargs)
:show-inheritance:

Search engine results
=====================

Expand Down
1 change: 1 addition & 0 deletions tests/test_pages.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ def test_matching_items():
"ProductNavigation": {"dateDownloaded", "validationMessages"},
"RealEstate": {"dateDownloaded", "probability", "validationMessages"},
"JobPosting": {"dateDownloaded", "probability", "searchText", "validationMessages"},
"JobPostingNavigation": {"dateDownloaded", "validationMessages"},
"Serp": {
"dateDownloaded",
"displayedQuery",
Expand Down
5 changes: 5 additions & 0 deletions zyte_common_items/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@
CustomAttributesValues,
JobPosting,
JobPostingMetadata,
JobPostingNavigation,
JobPostingNavigationMetadata,
Product,
ProductFromList,
ProductList,
Expand All @@ -78,6 +80,7 @@
AutoArticleNavigationPage,
AutoArticlePage,
AutoBusinessPlacePage,
AutoJobPostingNavigationPage,
AutoJobPostingPage,
AutoProductListPage,
AutoProductNavigationPage,
Expand All @@ -89,6 +92,7 @@
BaseArticleNavigationPage,
BaseArticlePage,
BaseBusinessPlacePage,
BaseJobPostingNavigationPage,
BaseJobPostingPage,
BasePage,
BaseProductListPage,
Expand All @@ -99,6 +103,7 @@
BaseSocialMediaPostPage,
BusinessPlacePage,
HasMetadata,
JobPostingNavigationPage,
JobPostingPage,
MetadataT,
Page,
Expand Down
1 change: 1 addition & 0 deletions zyte_common_items/items/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
CustomAttributesValues,
)
from .job_posting import JobPosting, JobPostingMetadata
from .job_posting_navigation import JobPostingNavigation, JobPostingNavigationMetadata
from .product import Product, ProductMetadata, ProductVariant
from .product_list import ProductFromList, ProductList, ProductListMetadata
from .product_navigation import ProductNavigation, ProductNavigationMetadata
Expand Down
46 changes: 46 additions & 0 deletions zyte_common_items/items/job_posting_navigation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
from typing import List, Optional

import attrs

from zyte_common_items.base import Item
from zyte_common_items.components import ListMetadata, ProbabilityRequest, Request
from zyte_common_items.converters import (
to_metadata_optional,
to_probability_request_list_optional,
url_to_str,
)


@attrs.define(kw_only=True)
class JobPostingNavigationMetadata(ListMetadata):
"""Metadata class for :data:`zyte_common_items.JobPostingNavigation.metadata`."""


@attrs.define(kw_only=True)
class JobPostingNavigation(Item):
"""Represents the navigational aspects of a job posting listing page on a
job website"""

#: Main URL from which the data is extracted.
url: str = attrs.field(converter=url_to_str)

#: List of job postings available on this page.
items: Optional[List[ProbabilityRequest]] = attrs.field(
default=None, converter=to_probability_request_list_optional, kw_only=True # type: ignore[misc]
)

#: A link to the next page, if available.
nextPage: Optional[Request] = None

#: Number of the current page.
#:
#: It should only be extracted if the webpage shows a page number.
#:
#: It must be 1-based. For example, if the first page of a listing is
#: numbered as 0 on the website, it should be extracted as `1` nonetheless.
pageNumber: Optional[int] = None

#: Data extraction process metadata.
metadata: Optional[JobPostingNavigationMetadata] = attrs.field(
default=None, converter=to_metadata_optional(JobPostingNavigationMetadata), kw_only=True # type: ignore[misc]
)
5 changes: 5 additions & 0 deletions zyte_common_items/pages/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@
BusinessPlacePage,
)
from .job_posting import AutoJobPostingPage, BaseJobPostingPage, JobPostingPage
from .job_posting_navigation import (
AutoJobPostingNavigationPage,
BaseJobPostingNavigationPage,
JobPostingNavigationPage,
)
from .mixins import DescriptionMixin, HasMetadata, MetadataT, PriceMixin
from .product import AutoProductPage, BaseProductPage, ProductPage
from .product_list import AutoProductListPage, BaseProductListPage, ProductListPage
Expand Down
53 changes: 53 additions & 0 deletions zyte_common_items/pages/job_posting_navigation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
from typing import List, Optional

import attrs
from web_poet import Returns

from zyte_common_items.components import ProbabilityRequest, Request
from zyte_common_items.fields import auto_field
from zyte_common_items.items import JobPostingNavigation, JobPostingNavigationMetadata
from zyte_common_items.processors import probability_request_list_processor

from .base import BasePage, Page
from .mixins import HasMetadata


class BaseJobPostingNavigationPage(
BasePage, Returns[JobPostingNavigation], HasMetadata[JobPostingNavigationMetadata]
):
""":class:`BasePage` subclass for :class:`JobPostingNavigation`."""

class Processors(BasePage.Processors):
subCategories = [probability_request_list_processor]
items = [probability_request_list_processor]


class JobPostingNavigationPage(
Page, Returns[JobPostingNavigation], HasMetadata[JobPostingNavigationMetadata]
):
""":class:`Page` subclass for :class:`JobPostingNavigation`."""


@attrs.define
class AutoJobPostingNavigationPage(BaseJobPostingNavigationPage):
job_posting_navigation: JobPostingNavigation

@auto_field
def items(self) -> Optional[List[ProbabilityRequest]]:
return self.job_posting_navigation.items

@auto_field
def metadata(self) -> Optional[JobPostingNavigationMetadata]:
return self.job_posting_navigation.metadata

@auto_field
def nextPage(self) -> Optional[Request]:
return self.job_posting_navigation.nextPage

@auto_field
def pageNumber(self) -> Optional[int]:
return self.job_posting_navigation.pageNumber

@auto_field
def url(self) -> Optional[str]:
return self.job_posting_navigation.url

0 comments on commit f3d6356

Please sign in to comment.