enh: listing source (#168)

Bunsly · Jul 16, 2024 · edffe18 · edffe18
1 parent 0988230
commit edffe18
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 4 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.57"
+version = "1.1.58"
 description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
 authors = ["Zachary Hampton <[email protected]>", "Cullen Watson <[email protected]>"]
 homepage = "https://github.com/Bunsly/JobSpy"

diff --git a/src/jobspy/__init__.py b/src/jobspy/__init__.py
@@ -209,6 +209,7 @@ def convert_to_annual(job_data: dict):
             "currency",
             "is_remote",
             "job_function",
+            "listing_type",
             "emails",
             "description",
             "company_url",

diff --git a/src/jobspy/jobs/__init__.py b/src/jobspy/jobs/__init__.py
@@ -242,6 +242,7 @@ class JobPost(BaseModel):
     date_posted: date | None = None
     emails: list[str] | None = None
     is_remote: bool | None = None
+    listing_type: str | None = None
 
     # indeed specific
     company_addresses: str | None = None

diff --git a/src/jobspy/scrapers/glassdoor/__init__.py b/src/jobspy/scrapers/glassdoor/__init__.py
@@ -189,7 +189,15 @@ def _process_job(self, job_data):
         except:
             description = None
         company_url = f"{self.base_url}Overview/W-EI_IE{company_id}.htm"
-        company_logo = job_data["jobview"].get("overview", {}).get("squareLogoUrl", None)
+        company_logo = (
+            job_data["jobview"].get("overview", {}).get("squareLogoUrl", None)
+        )
+        listing_type = (
+            job_data["jobview"]
+            .get("header", {})
+            .get("adOrderSponsorshipLevel", "")
+            .lower()
+        )
         return JobPost(
             id=str(job_id),
             title=title,
@@ -203,6 +211,7 @@ def _process_job(self, job_data):
             description=description,
             emails=extract_emails_from_text(description) if description else None,
             logo_photo_url=company_logo,
+            listing_type=listing_type,
         )
 
     def _fetch_job_description(self, job_id):

diff --git a/src/jobspy/scrapers/indeed/__init__.py b/src/jobspy/scrapers/indeed/__init__.py
@@ -176,7 +176,7 @@ def _build_filters(self):
                 keys.append("DSQF7")
 
             if keys:
-                keys_str = '", "'.join(keys)  # Prepare your keys string
+                keys_str = '", "'.join(keys)
                 filters_str = f"""
                 filters: {{
                   composite: {{
@@ -353,7 +353,6 @@ def _get_compensation_interval(interval: str) -> CompensationInterval:
           jobSearch(
             {what}
             {location}
-            includeSponsoredResults: NONE
             limit: 100
             sort: DATE
             {cursor}
@@ -365,6 +364,9 @@ def _get_compensation_interval(interval: str) -> CompensationInterval:
             results {{
               trackingKey
               job {{
+                source {{
+                  name
+                }}
                 key
                 title
                 datePublished

diff --git a/src/jobspy/scrapers/ziprecruiter/__init__.py b/src/jobspy/scrapers/ziprecruiter/__init__.py
@@ -135,6 +135,7 @@ def _process_job(self, job: dict) -> JobPost | None:
         self.seen_urls.add(job_url)
 
         description = job.get("job_description", "").strip()
+        listing_type = job.get("buyer_type", "")
         description = (
             markdown_converter(description)
             if self.scraper_input.description_format == DescriptionFormat.MARKDOWN
@@ -175,6 +176,7 @@ def _process_job(self, job: dict) -> JobPost | None:
             description=description_full if description_full else description,
             emails=extract_emails_from_text(description) if description else None,
             job_url_direct=job_url_direct,
+            listing_type=listing_type,
         )
 
     def _get_descr(self, job_url):