Browse Source

improve seasons/episodes fetching

thezero 3 years ago
parent
commit
913be0e412
4 changed files with 47 additions and 25 deletions
  1. 12 9
      api.py
  2. 4 5
      db.py
  3. 5 5
      db_to_html.py
  4. 26 6
      vvvvget.py

+ 12 - 9
api.py

@@ -67,27 +67,28 @@ class Api():
 		return self.stream_url
 
 	def login(self):
-		login = requests.get('https://www.vvvvid.it/user/login', headers={'User-Agent': Api.ua})
-		self.conn_id = login.json()['data']['conn_id']
+		login = requests.get('https://www.vvvvid.it/user/login', headers={'User-Agent': Api.ua}).json()
+		self.conn_id = login['data']['conn_id']
 
 	def get_info(self, show_id):
 		info = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/info/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua})
 		info.encoding = 'utf-8'
-		if info.json()['result'] == 'ok':
-			return info.json()['data']
+		info = info.json()
+		if info['result'] == 'ok':
+			return info['data']
 		else:
 			return False
 
 	def get_seasons(self, show_id):
-		seasons = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/seasons/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua})
-		if seasons.json()['result'] == 'ok' and seasons.json()['data'] and seasons.json()['data'][0]['episodes']:
-			return seasons.json()['data']
+		seasons = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/seasons/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}).json()
+		if seasons['result'] == 'ok' and seasons['data']:  # and seasons['data'][0]['episodes']:
+			return seasons['data']
 		else:
 			return []
 
 	def get_episodes(self, season_id, show_id):
 		episodes = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/season/' + str(season_id) + '?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}).json()
-		if episodes['result'] == 'ok' and episodes['data'] and episodes['data'][0]['embed_info']:
+		if episodes['result'] == 'ok' and episodes['data']:  # and episodes['data'][0]['embed_info']:
 			return episodes['data']
 		else:
 			return []
@@ -97,7 +98,7 @@ class Api():
 		count = 0
 		for k in episodes:
 			count += 1
-			if k['embed_info']:
+			if k.get('embed_info'):
 				if k['video_type'] == 'video/rcs':
 					embed_info = self.ds(k['embed_info'])
 					embed_info = 'https' + embed_info[4:30] + 'i' + embed_info[31:-12] + 'master.m3u8'
@@ -109,5 +110,7 @@ class Api():
 					embed_info = self.ds(k['embed_info'])
 				else:
 					embed_info = self.ds(k['embed_info'])
+			else:
+				embed_info = ""
 			eps.append((count, show_id, season_id, k['video_type'], embed_info))
 		return eps

+ 4 - 5
db.py

@@ -34,7 +34,7 @@ class VVVVIDatabase():
 		con.close()
 		# getting series in a useful format for later
 		series = {}
-		for i,s in rows:
+		for i, s in rows:
 			if series.get(i) is None:
 				series[i] = []
 			series[i].append(s)
@@ -49,12 +49,11 @@ class VVVVIDatabase():
 		con.close()
 		# getting series in a useful format for later
 		series = {}
-		for i,s in rows:
+		for i, s in rows:
 			if series.get(i) is None:
 				series[i] = []
 			series[i].append(s)
-		return series		
-
+		return series
 
 	def insert_serie(self, serie):
 		con = sqlite3.connect(self.path)
@@ -80,4 +79,4 @@ class VVVVIDatabase():
 		except sqlite3.IntegrityError:
 			# episodi gia' presenti
 			pass
-		return False
+		return False

+ 5 - 5
db_to_html.py

@@ -3,7 +3,7 @@ import sqlite3
 
 vvvvidb = "vvvvidb.sqlite3"
 
-if not os.path.isfile(vvvvidb): 
+if not os.path.isfile(vvvvidb):
     print("{} file not found. Scrape the DB first using vvvvget.py".format(vvvvidb))
     exit(1)
 
@@ -23,10 +23,10 @@ out.write('''<!DOCTYPE html>
 <body>
 <a href="''' + vvvvidb + '''">Source Database</a><br/><br/>''')
 
-# select all the series
+# Select all the series
 # cur.execute("SELECT * FROM series;")
 
-# selecy only series with at least one episodes
+# Selecy only series with at least one episodes
 cur.execute("SELECT s.* FROM series as s INNER JOIN episodes as e ON e.serie_id = s.id AND e.season_id = s.season_id GROUP BY s.id, s.season_id;")
 series = cur.fetchall()
 
@@ -34,7 +34,7 @@ for s in series:
     serie = (str(s[0]), str(s[2]))
     out.write("\t<h3>{}: {} - Id Serie: {} - Tipo: {}</h3>\n\t<ul>\n".format(s[0], s[1], s[2], s[3]))
 
-    cur.execute("SELECT e.cdn_url FROM episodes AS e JOIN series AS s ON e.serie_id = s.id AND e.season_id = s.season_id WHERE s.id = ? AND s.season_id = ?;", serie)
+    cur.execute("SELECT e.cdn_url FROM episodes AS e JOIN series AS s ON e.serie_id = s.id AND e.season_id = s.season_id WHERE s.id = ? AND s.season_id = ? AND e.cdn_url != '';", serie)
     episodes = cur.fetchall()
 
     [out.write("\t\t<li><a href=\"player.html#{0}\">{0}</a></li>\n".format(e[0])) for e in episodes]
@@ -44,5 +44,5 @@ for s in series:
 out.write('\n</body>\n</html>')
 out.close()
 
-con.commit() 
+con.commit()
 con.close()

+ 26 - 6
vvvvget.py

@@ -1,7 +1,3 @@
-import os
-import json
-import requests
-import sqlite3
 from db import VVVVIDatabase
 from api import Api
 
@@ -13,6 +9,8 @@ if stream_url is None:
 
 vvvvidb = VVVVIDatabase("vvvvidb.sqlite3")
 api.login()
+last = 0
+smax = 1000
 
 if not vvvvidb.is_valid:
 	# Database file not present
@@ -24,9 +22,31 @@ else:
 	# We scan all the older id to see if there are new episodes, then we scan every id greater then the last one
 	ids = vvvvidb.series_id()
 
-last = 0
+	for i in ids.keys():
+		print("Fetching old...{}".format(i))
+		info = api.get_info(i)
+		if not info:
+			continue
+		seasons = api.get_seasons(i)
+		for j in seasons:
+			if j['season_id'] not in ids.get(i):
+				# We found a new season for an old show
+				if not vvvvidb.insert_serie((info['show_id'], info['title'], j['season_id'], j['name'])):
+					continue
+
+			# New episodes for a new or old season
+			episodes = api.get_episodes(j['season_id'], i)
+			eps = api.format_episodes(j['season_id'], info['show_id'], episodes)
+			print("Found {} episodes".format(len(eps)))
+			vvvvidb.insert_episodes(eps)
+
+		last = i + 1
+		smax = 200
+
+	print("Resuming from...{}".format(last))
+
 # Scan all the episodes
-for i in range(last, min(last + 500, 1000)):
+for i in range(last, last + smax):
 	print("Fetching...{}".format(i))
 	info = api.get_info(i)
 	if not info: