Updating the xbvr scraper

scrapers/xbvrdb.py

@@ -1,7 +1,20 @@
 import json
 import sys
 import sqlite3
-from os import path
+import os
+import requests
+
+try:
+    import py_common.graphql as graphql
+    import py_common.log as log
+except ModuleNotFoundError:
+    print("You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)", file=sys.stderr)
+    sys.exit()
+
+# Set XBVR_HOST to your xbvr instance, if set to None it will use xbvr.db and query that instead
+XBVR_HOST='http://192.168.0.35:9999'
+#XBVR_HOST=None
+
 
 ''' This script uses the sqlite database from xbvr (3d porn manager) 
     Copy main.db from yout xbvr configuration and rename this to xbvr.db
@@ -10,7 +23,7 @@
    '''
 def lookup_scene(id):
     c=conn.cursor()
-    c.execute('select title,synopsis,site,cover_url,scene_url,date(release_date, "localtime") from scenes where id=?',(id,))
+    c.execute('SELECT title,synopsis,site,cover_url,scene_url,date(release_date, "localtime") FROM scenes WHERE id=?',(id,))
     row=c.fetchone()
     res={}
     res['title']=row[0]
@@ -19,20 +32,24 @@ def lookup_scene(id):
     res['image']=row[3]
     res['url']=row[4]
     res['date']=row[5]
-    c.execute("select tags.name from scene_tags,tags where scene_tags.tag_id=tags.id and scene_tags.scene_id=? ;",(id,))
+    c.execute("SELECT tags.name FROM scene_tags,tags WHERE scene_tags.tag_id=tags.id AND scene_tags.scene_id=? ;",(id,))
     row = c.fetchall()
     res['tags']=[{"name":x[0]} for x in row]
-    c.execute("select actors.name from scene_cast,actors where actors.id=scene_cast.actor_id and scene_cast.scene_id=? ;",(id,))
+    c.execute("SELECT actors.name FROM scene_cast,actors WHERE actors.id=scene_cast.actor_id AND scene_cast.scene_id=? ;",(id,))
     row = c.fetchall()
     res['performers']=[{"name":x[0]} for x in row]
     return res
 
 def find_scene_id(title):
     c = conn.cursor()
-    c.execute('SELECT scene_id FROM files WHERE filename=?', (title,))
+    c.execute('SELECT scene_id FROM files WHERE filename LIKE ?', (title,))
     id=c.fetchone()
     if id == None:
-        c.execute('SELECT id FROM scenes WHERE scene_id=?', (title,))
+        c.execute('SELECT scene_id FROM files WHERE filename LIKE ?', (title+'____',))
+        id=c.fetchone()
+        if id is not None:
+            return id[0]
+        c.execute('SELECT id FROM scenes WHERE scene_id LIKE ?', (title,))
         id = c.fetchone()
         if id is not None:
             return id[0]
@@ -48,26 +65,76 @@ def find_scene_id(title):
             if 'originals' in t:
                 t.remove('originals')
             title='%'.join(t)+'%'
-        c.execute('select id from scenes where title like ?',(title+'%',))
+        c.execute('SELECT id FROM scenes WHERE title LIKE ?',(title+'%',))
         id=c.fetchone()
         if id is not None:
             return id[0]
     else:
         return id[0]
     return None
 
-if not path.exists("xbvr.db"):
+def query_api(filename):
+    request_config={"dlState":"available","cardSize":"1","lists":[],"isAvailable":True,"isAccessible":True,"isHidden":False,"isWatched":None,"releaseMonth":"","cast":[],"sites":[],"tags":[],"cuepoint":[],"attributes":[],"volume":0,"sort":"release_desc","offset":0,"limit":1}
+    response = requests.post(XBVR_HOST+'/api/scene/list', json=request_config)
+    if response.status_code == 200:
+        total_scenes = response.json()['results']
+        log.debug('total scenes %s' % (total_scenes))
+        request_config['limit'] = 500
+        request_config['offset'] = 0
+        while request_config['offset'] < total_scenes:
+            response = requests.post(XBVR_HOST+'/api/scene/list', json=request_config)
+            for s in response.json()['scenes']:
+                for f in s['file']:
+                    if f['filename'] == filename:
+
+                       result={
+                               'title':s['title'],
+                               'details':s['synopsis'],
+                               'studio':{'name':s['site']},
+                               'image':s['cover_url'],
+                               'url':s['scene_url'],
+                               'date':s['release_date_text'],
+                               'tags':[{"name":x['name']} for x in s['tags']],
+                               'performers':[{"name":x['name']} for x in s['cast']]
+                               }
+                       log.debug(result)
+                       print(json.dumps(result))
+                       exit(0)
+            request_config['offset'] = request_config['offset'] + request_config['limit']
+
+
+
+
+
+if XBVR_HOST is not None:
+    if sys.argv[1] == "query":
+        fragment = json.loads(sys.stdin.read())
+        response=graphql.getScene(fragment['id'])
+        filename = os.path.basename(response['path'])
+        query_api(filename)
+
+    exit(0)
+
+if not os.path.exists("xbvr.db"):
     print("Error, the sqlite database xbvr.db does not exist in the scrapers directory.",file=sys.stderr)
     print("Copy this database from the docker container and give it the name xbvr.db",file=sys.stderr)
     print("docker cp xbvr:/root/.config/xbvr/main.db xbvr.db",file=sys.stderr)
     exit(1)
 
-
 conn = sqlite3.connect('xbvr.db',detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
 
 if sys.argv[1] == "query":
     fragment = json.loads(sys.stdin.read())
     print(json.dumps(fragment),file=sys.stderr)
+    response=graphql.getScene(fragment['id'])
+    filename = os.path.basename(response['path'])
+    scene_id=find_scene_id(filename)
+    if scene_id:
+        result=lookup_scene(scene_id)
+        print(json.dumps(result))
+
+        conn.close()
+        exit(0)
     scene_id = find_scene_id(fragment['title'])
     if not scene_id:
         print(f"Could not determine scene id in title: `{fragment['title']}`",file=sys.stderr)
@@ -90,3 +157,4 @@ def find_scene_id(title):
         result.pop("image",None)
         print(json.dumps(result))
     conn.close()
+

scrapers/xbvrdb.yml

@@ -12,4 +12,4 @@ galleryByFragment:
     - python3
     - xbvrdb.py
     - gallery_query
-# Last Updated March 13, 2022
+# Last Updated May 07, 2023